mirror of
https://github.com/yacy/yacy_search_server.git
synced 2026-10-25 05:54:15 +00:00
Enhanced Log Reports
This commit is contained in:
1 parent
adb04ae397
commit
9f540de067
6 files changed
+326
-163
No files matched your search
+16
-17
@@ -1437,23 +1437,22 @@ decoration.grafics.linkstructure = true
|
||||
decoration.hostanalysis = false
|
||||
decoration.simpleheadernavbar = navbar-default
|
||||
|
||||
# ai settings
|
||||
ai.production_models = []
|
||||
ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.
|
||||
ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.
|
||||
ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n
|
||||
ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array:
|
||||
ai.logreport.enabled = true
|
||||
ai.logreport.dir = DATA/REPORTS/log
|
||||
ai.logreport.initial_delay_minutes = 5
|
||||
ai.logreport.period_minutes = 60
|
||||
ai.logreport.max_bucket_lines = 100000
|
||||
ai.logreport.max_tokens = 4096
|
||||
ai.logreport.daily_compression.enabled = true
|
||||
ai.logreport.daily_compression.delete_hourly = false
|
||||
ai.logreport.feed.max_entries = 100
|
||||
ai.shield.allow-nonlocalhost = false
|
||||
ai.shield.show-chat-link = false
|
||||
# ai settings
|
||||
ai.production_models = []
|
||||
ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.
|
||||
ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.
|
||||
ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n
|
||||
ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array:
|
||||
ai.logreport.enabled = true
|
||||
ai.logreport.dir = DATA/REPORTS/log
|
||||
ai.logreport.initial_delay_minutes = 5
|
||||
ai.logreport.period_minutes = 60
|
||||
ai.logreport.max_bucket_lines = 100000
|
||||
ai.logreport.max_tokens = 4096
|
||||
ai.logreport.daily_compression.enabled = true
|
||||
ai.logreport.feed.max_entries = 100
|
||||
ai.shield.allow-nonlocalhost = false
|
||||
ai.shield.show-chat-link = false
|
||||
ai.shield.rate.per-minute = 4
|
||||
ai.shield.rate.per-hour = 60
|
||||
ai.shield.rate.per-day = 120
|
||||
|
||||
+155
-18
@@ -3,10 +3,59 @@
|
||||
<head>
|
||||
<title>YaCy '#[clientname]#': Log Reports</title>
|
||||
#%env/templates/metas.template%#
|
||||
<link rel="stylesheet" href="js/styles/a11y-dark.min.css" type="text/css" />
|
||||
<link rel="stylesheet" href="env/markdown.css" type="text/css" />
|
||||
<style type="text/css">
|
||||
.logreport-feeds {
|
||||
margin: 0 0 16px 0;
|
||||
}
|
||||
/* two-column layout: navigation of existing reports left, selected report right */
|
||||
.logreport-layout {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 16px;
|
||||
}
|
||||
.logreport-nav {
|
||||
flex: 0 0 260px;
|
||||
max-height: 640px;
|
||||
overflow: auto;
|
||||
border: 1px solid #ddd;
|
||||
background: #fff;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
.logreport-nav li {
|
||||
border-bottom: 1px solid #eee;
|
||||
}
|
||||
.logreport-nav a {
|
||||
display: block;
|
||||
padding: 8px 10px;
|
||||
text-decoration: none;
|
||||
color: #333;
|
||||
}
|
||||
.logreport-nav a:hover {
|
||||
background: #f5f5f5;
|
||||
}
|
||||
.logreport-nav li.selected a {
|
||||
background: #eef4fa;
|
||||
border-left: 4px solid #337ab7;
|
||||
padding-left: 6px;
|
||||
font-weight: bold;
|
||||
}
|
||||
.logreport-nav .nav-date {
|
||||
display: block;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
.logreport-nav .nav-type {
|
||||
display: block;
|
||||
color: #888;
|
||||
font-size: 0.8em;
|
||||
}
|
||||
.logreport-main {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
}
|
||||
.logreport-card {
|
||||
border: 1px solid #ddd;
|
||||
margin: 0 0 16px 0;
|
||||
@@ -38,6 +87,16 @@
|
||||
font-size: 12px;
|
||||
line-height: 1.45;
|
||||
}
|
||||
/* markdown output styles come from the shared env/markdown.css (.markdown-body);
|
||||
only the report container itself is styled locally. white-space must be
|
||||
repeated here: the .logreport-body pre-wrap rule (raw markdown fallback)
|
||||
would otherwise win over the shared .markdown-body rule, which renders the
|
||||
newlines between the generated HTML blocks as visible empty lines */
|
||||
.logreport-body.markdown-body {
|
||||
max-width: 100%;
|
||||
padding: 12px;
|
||||
white-space: normal;
|
||||
}
|
||||
.logreport-empty {
|
||||
border: 1px solid #ddd;
|
||||
background: #f9f9f9;
|
||||
@@ -54,6 +113,28 @@
|
||||
.logreport-actions {
|
||||
margin: 0 0 16px 0;
|
||||
}
|
||||
.logreport-progress {
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
border: 1px solid #5bc0de;
|
||||
border-left: 6px solid #5bc0de;
|
||||
background: #eef9fc;
|
||||
color: #245269;
|
||||
padding: 12px;
|
||||
margin: 10px 0 16px 0;
|
||||
}
|
||||
.logreport-progress .spinner {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
border: 3px solid #b7dff0;
|
||||
border-top-color: #31708f;
|
||||
border-radius: 50%;
|
||||
animation: logreport-spin 0.8s linear infinite;
|
||||
}
|
||||
@keyframes logreport-spin {
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
.logreport-message {
|
||||
border: 1px solid #5bc0de;
|
||||
border-left: 6px solid #5bc0de;
|
||||
@@ -70,18 +151,31 @@
|
||||
|
||||
<h2>Log Reports</h2>
|
||||
|
||||
<form action="LogReports_p.html" method="get" accept-charset="UTF-8">
|
||||
<fieldset>
|
||||
<label for="count">Reports</label>
|
||||
<input type="number" name="count" id="count" min="0" max="#[maxcount]#" value="#[count]#" />
|
||||
<input type="submit" value="refresh" />
|
||||
</fieldset>
|
||||
<form action="LogReports_p.html" method="post" accept-charset="UTF-8" class="logreport-actions" id="runReportForm">
|
||||
<button type="submit" name="runReportNow" value="1" class="btn btn-primary" id="runReportButton">run report now</button>
|
||||
</form>
|
||||
|
||||
<form action="LogReports_p.html" method="post" accept-charset="UTF-8" class="logreport-actions">
|
||||
<input type="hidden" name="count" value="#[count]#" />
|
||||
<button type="submit" name="runReportNow" value="1" class="btn btn-primary">run report now</button>
|
||||
</form>
|
||||
<!-- shown while the synchronous report generation request is running; the page
|
||||
reloads with the result when the server is done, which removes this again -->
|
||||
<div class="logreport-progress" id="runReportProgress">
|
||||
<div class="spinner"></div>
|
||||
<div>Generating report from the current-hour log lines — the LLM call can take a while …
|
||||
<span id="runReportElapsed">0</span> seconds elapsed</div>
|
||||
</div>
|
||||
|
||||
<script type="text/javascript">
|
||||
document.getElementById("runReportForm").addEventListener("submit", function () {
|
||||
var button = document.getElementById("runReportButton");
|
||||
// a submit button disabled during the submit event would drop its
|
||||
// runReportNow=1 value from the request, so disable it one tick later
|
||||
setTimeout(function () { button.disabled = true; }, 0);
|
||||
document.getElementById("runReportProgress").style.display = "flex";
|
||||
var started = Date.now();
|
||||
setInterval(function () {
|
||||
document.getElementById("runReportElapsed").textContent = Math.floor((Date.now() - started) / 1000);
|
||||
}, 1000);
|
||||
});
|
||||
</script>
|
||||
|
||||
#(runReportResult)#
|
||||
::
|
||||
@@ -119,23 +213,66 @@
|
||||
<div class="logreport-empty">The report directory does not exist yet. Reports will appear here after the scheduler has generated the first completed hourly report.</div>
|
||||
#(/reportdirMissing)#
|
||||
|
||||
#{reports}#
|
||||
<div class="logreport-card">
|
||||
<div class="logreport-header">
|
||||
<div class="logreport-title">#[title]#</div>
|
||||
<div class="logreport-meta">
|
||||
#[type]# report - #[filename]# - #[published]# - #[chars]# characters
|
||||
#(hasReports)#
|
||||
::
|
||||
<div class="logreport-layout">
|
||||
<ul class="logreport-nav">
|
||||
#{reports}#
|
||||
<li #(selected)#::class="selected"#(/selected)#>
|
||||
<a href="LogReports_p.html?report=#[filename]#" title="#[filename]#">
|
||||
<span class="nav-date">#[date]#</span>
|
||||
<span class="nav-type">#[type]# report</span>
|
||||
</a>
|
||||
</li>
|
||||
#{/reports}#
|
||||
</ul>
|
||||
|
||||
<div class="logreport-main">
|
||||
#(selectedReport)#
|
||||
::
|
||||
<div class="logreport-card">
|
||||
<div class="logreport-header">
|
||||
<div class="logreport-title">#[title]#</div>
|
||||
<div class="logreport-meta">
|
||||
#[type]# report - #[filename]# - #[published]# - #[chars]# characters
|
||||
</div>
|
||||
</div>
|
||||
<!-- the template writes the escaped markdown source into this element; the
|
||||
script below replaces it with rendered markdown (same pipeline as the
|
||||
chat messages in yacychat.html) and adds the markdown-body class. Without
|
||||
JavaScript the raw markdown stays readable through the pre-wrap rule -->
|
||||
<div class="logreport-body" id="reportMarkdown">#[content]#</div>
|
||||
</div>
|
||||
#(/selectedReport)#
|
||||
</div>
|
||||
<pre class="logreport-body">#[content]#</pre>
|
||||
</div>
|
||||
#{/reports}#
|
||||
#(/hasReports)#
|
||||
|
||||
#(noReports)#
|
||||
::
|
||||
<div class="logreport-empty">No generated log reports were found.</div>
|
||||
#(/noReports)#
|
||||
|
||||
<script src="js/highlight.min.js"></script>
|
||||
<script src="js/marked.umd.js"></script>
|
||||
<script src="js/yacy-markdown.js"></script>
|
||||
<script type="text/javascript">
|
||||
// Render the selected report with the shared markdown pipeline (js/yacy-markdown.js),
|
||||
// the same one used for the chat messages in yacychat.html. textContent yields the
|
||||
// raw markdown source which the server-side template wrote in HTML-escaped form.
|
||||
(function () {
|
||||
var container = document.getElementById("reportMarkdown");
|
||||
if (!container || typeof marked === "undefined") return;
|
||||
// breaks:false because report files have hard-wrapped source lines which
|
||||
// must flow as continuous text, not render as one <br> per source line
|
||||
container.innerHTML = YaCyMarkdown.render(container.textContent, { breaks: false });
|
||||
// the class switches the container from the pre-wrap raw-text fallback
|
||||
// to the rendered-markdown styles from env/markdown.css
|
||||
container.classList.add("markdown-body");
|
||||
YaCyMarkdown.highlightAll(container);
|
||||
})();
|
||||
</script>
|
||||
|
||||
#%env/templates/footer.template%#
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* yacy-markdown.js
|
||||
* Shared markdown rendering helpers, used by yacychat.html and LogReports_p.html.
|
||||
* Renders LLM-generated markdown to sanitized HTML: marked for parsing, a
|
||||
* sanitizer that strips active content (the source is model output, not trusted
|
||||
* input), and highlight.js for fenced code blocks. Style rules for the rendered
|
||||
* output live in env/markdown.css, scoped to the .markdown-body class.
|
||||
* Requires js/marked.umd.js; js/highlight.min.js is optional.
|
||||
*/
|
||||
(function (global) {
|
||||
"use strict";
|
||||
|
||||
const markedOptions = {
|
||||
gfm: true,
|
||||
breaks: true,
|
||||
smartLists: true,
|
||||
mangle: false,
|
||||
headerIds: false
|
||||
};
|
||||
|
||||
function escapeHTML(value) {
|
||||
const div = document.createElement('div');
|
||||
div.textContent = value || '';
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
function sanitizeHTML(html) {
|
||||
if (!html) return '';
|
||||
const template = document.createElement('template');
|
||||
template.innerHTML = html;
|
||||
const blockedTags = new Set(['script', 'style', 'iframe', 'object', 'embed', 'link', 'meta']);
|
||||
const walker = document.createTreeWalker(template.content, NodeFilter.SHOW_ELEMENT, null);
|
||||
const toRemove = [];
|
||||
while (walker.nextNode()) {
|
||||
const el = walker.currentNode;
|
||||
if (!el || !el.tagName) continue;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (blockedTags.has(tag)) {
|
||||
toRemove.push(el);
|
||||
continue;
|
||||
}
|
||||
for (const attr of Array.from(el.attributes)) {
|
||||
const name = attr.name.toLowerCase();
|
||||
const value = attr.value || '';
|
||||
if (name.startsWith('on')) {
|
||||
el.removeAttribute(attr.name);
|
||||
continue;
|
||||
}
|
||||
if ((name === 'href' || name === 'src') && /^\s*javascript:/i.test(value)) {
|
||||
el.removeAttribute(attr.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
toRemove.forEach(node => node.remove());
|
||||
return template.innerHTML;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render markdown source to sanitized HTML; falls back to escaped plain text
|
||||
* when marked is not available or parsing fails.
|
||||
* optionOverrides can adjust the shared marked options per call, e.g.
|
||||
* { breaks: false } for documents with hard-wrapped source lines where a
|
||||
* single newline must not become a visible line break.
|
||||
*/
|
||||
function render(source, optionOverrides) {
|
||||
const text = typeof source === 'string' ? source : '';
|
||||
if (typeof marked === 'undefined') return escapeHTML(text);
|
||||
try {
|
||||
marked.setOptions(Object.assign({}, markedOptions, optionOverrides || {}));
|
||||
return sanitizeHTML(marked.parse(text));
|
||||
} catch (err) {
|
||||
console.warn('Markdown rendering failed', err);
|
||||
return escapeHTML(text);
|
||||
}
|
||||
}
|
||||
|
||||
/** Apply highlight.js to all fenced code blocks below the given container. */
|
||||
function highlightAll(container) {
|
||||
if (!container || typeof hljs === 'undefined') return;
|
||||
hljs.configure({ ignoreUnescapedHTML: true });
|
||||
container.querySelectorAll('pre code').forEach(block => hljs.highlightElement(block));
|
||||
}
|
||||
|
||||
global.YaCyMarkdown = { markedOptions, escapeHTML, sanitizeHTML, render, highlightAll };
|
||||
})(window);
|
||||
+9
-85
@@ -4,6 +4,7 @@
|
||||
<title>YaCy '#[clientname]#': Chat</title>
|
||||
#%env/templates/metas.template%#
|
||||
<link rel="stylesheet" href="js/styles/a11y-dark.min.css" type="text/css" />
|
||||
<link rel="stylesheet" href="env/markdown.css" type="text/css" />
|
||||
<style type="text/css">
|
||||
/* 1. Global Container */
|
||||
.chat-panel {
|
||||
@@ -121,16 +122,9 @@
|
||||
font-family: 'Roboto Mono', 'Menlo', 'Consolas', monospace;
|
||||
}
|
||||
|
||||
.chat-body.markdown {
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.chat-body.markdown,
|
||||
.chat-body.markdown * {
|
||||
font-family: 'Roboto Mono', 'Menlo', 'Consolas', monospace !important; /* Force markdown output and descendants to stay monospace, FTW! */
|
||||
}
|
||||
|
||||
.chat-body.markdown.markdown-body {
|
||||
/* markdown output styles come from the shared env/markdown.css (.markdown-body);
|
||||
only the chat-specific container properties are set here */
|
||||
.chat-body.markdown-body {
|
||||
max-width: 100%;
|
||||
padding: 0;
|
||||
}
|
||||
@@ -224,38 +218,6 @@
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.chat-body.markdown h1,
|
||||
.chat-body.markdown h2,
|
||||
.chat-body.markdown h3,
|
||||
.chat-body.markdown h4,
|
||||
.chat-body.markdown h5,
|
||||
.chat-body.markdown h6,
|
||||
.chat-body.markdown p,
|
||||
.chat-body.markdown li,
|
||||
.chat-body.markdown pre,
|
||||
.chat-body.markdown code {
|
||||
font-size: 1.2rem;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.chat-body pre {
|
||||
background-color: #2c3e50;
|
||||
border: 1px solid #dfe2e5;
|
||||
border-radius: 4px;
|
||||
padding: 2px;
|
||||
overflow-x: auto;
|
||||
font-family: 'Roboto Mono', 'Menlo', 'Consolas', monospace;
|
||||
white-space: pre;
|
||||
}
|
||||
|
||||
.chat-body code {
|
||||
font-family: 'Roboto Mono', 'Menlo', 'Consolas', monospace;
|
||||
font-size: 0.95rem;
|
||||
background-color: rgba(27,31,35,0.05);
|
||||
padding: 2px 4px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
/* 6. Composer Row (inside User fieldset) */
|
||||
.composer {
|
||||
display: flex;
|
||||
@@ -816,6 +778,7 @@
|
||||
|
||||
<script src="js/highlight.min.js"></script>
|
||||
<script src="js/marked.umd.js"></script>
|
||||
<script src="js/yacy-markdown.js"></script>
|
||||
<script src="js/index.umd.min.js"></script>
|
||||
<script src="js/vfs.js"></script>
|
||||
<script type="text/javascript">
|
||||
@@ -996,52 +959,13 @@
|
||||
} catch (err) {
|
||||
console.warn('Failed to initialize syntax highlighting', err);
|
||||
}
|
||||
marked.setOptions({
|
||||
gfm: true,
|
||||
breaks: true,
|
||||
smartLists: true,
|
||||
mangle: false,
|
||||
headerIds: false
|
||||
});
|
||||
marked.setOptions(YaCyMarkdown.markedOptions);
|
||||
return { enabled: true };
|
||||
})();
|
||||
|
||||
function escapeHTML(value) {
|
||||
const div = document.createElement('div');
|
||||
div.textContent = value || '';
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
function sanitizeHTML(html) {
|
||||
if (!html) return '';
|
||||
const template = document.createElement('template');
|
||||
template.innerHTML = html;
|
||||
const blockedTags = new Set(['script', 'style', 'iframe', 'object', 'embed', 'link', 'meta']);
|
||||
const walker = document.createTreeWalker(template.content, NodeFilter.SHOW_ELEMENT, null);
|
||||
const toRemove = [];
|
||||
while (walker.nextNode()) {
|
||||
const el = walker.currentNode;
|
||||
if (!el || !el.tagName) continue;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (blockedTags.has(tag)) {
|
||||
toRemove.push(el);
|
||||
continue;
|
||||
}
|
||||
for (const attr of Array.from(el.attributes)) {
|
||||
const name = attr.name.toLowerCase();
|
||||
const value = attr.value || '';
|
||||
if (name.startsWith('on')) {
|
||||
el.removeAttribute(attr.name);
|
||||
continue;
|
||||
}
|
||||
if ((name === 'href' || name === 'src') && /^\s*javascript:/i.test(value)) {
|
||||
el.removeAttribute(attr.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
toRemove.forEach(node => node.remove());
|
||||
return template.innerHTML;
|
||||
}
|
||||
// shared markdown helpers from js/yacy-markdown.js, also used by LogReports_p.html
|
||||
const escapeHTML = YaCyMarkdown.escapeHTML;
|
||||
const sanitizeHTML = YaCyMarkdown.sanitizeHTML;
|
||||
|
||||
function renderMarkdown(content) {
|
||||
const source = typeof content === 'string' ? content : '';
|
||||
|
||||
@@ -63,7 +63,6 @@ public class LogReportService {
|
||||
public static final String CONFIG_PERIOD_MINUTES = "ai.logreport.period_minutes";
|
||||
public static final String CONFIG_MAX_TOKENS = "ai.logreport.max_tokens";
|
||||
public static final String CONFIG_DAILY_COMPRESSION_ENABLED = "ai.logreport.daily_compression.enabled";
|
||||
public static final String CONFIG_DELETE_HOURLY_AFTER_DAILY = "ai.logreport.daily_compression.delete_hourly";
|
||||
public static final String CONFIG_FEED_MAX_ENTRIES = "ai.logreport.feed.max_entries";
|
||||
public static final String DEFAULT_REPORT_DIR = "DATA/REPORTS/log";
|
||||
public static final int DEFAULT_MAX_BUCKET_LINES = 100000;
|
||||
@@ -348,21 +347,24 @@ public class LogReportService {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
final Map<LocalDate, List<File>> completeDays = completeHourlyReportSets(reportDirectory);
|
||||
log.info("runId=" + runId + " event=daily-reports phase=scan completeDays=" + completeDays.size() + " path=" + reportDirectory.getAbsolutePath());
|
||||
if (completeDays.isEmpty()) {
|
||||
log.info("runId=" + runId + " event=daily-reports phase=skip reason=no-complete-day durationMs=" + elapsed(start));
|
||||
final Map<LocalDate, List<File>> pastDays = pastDayHourlyReportSets(reportDirectory);
|
||||
log.info("runId=" + runId + " event=daily-reports phase=scan pastDays=" + pastDays.size() + " path=" + reportDirectory.getAbsolutePath());
|
||||
if (pastDays.isEmpty()) {
|
||||
log.info("runId=" + runId + " event=daily-reports phase=skip reason=no-past-day-hourly-reports durationMs=" + elapsed(start));
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
final List<File> writtenReports = new ArrayList<>();
|
||||
int skippedExisting = 0;
|
||||
int failedReports = 0;
|
||||
for (final Map.Entry<LocalDate, List<File>> day : completeDays.entrySet()) {
|
||||
for (final Map.Entry<LocalDate, List<File>> day : pastDays.entrySet()) {
|
||||
final File dailyReportFile = new File(reportDirectory, dailyReportFilename(day.getKey()));
|
||||
if (dailyReportFile.exists()) {
|
||||
// the day was already compressed earlier; only clean up leftover hourly
|
||||
// reports (e.g. written by a manual "run report now" after the compression)
|
||||
skippedExisting++;
|
||||
log.info("runId=" + runId + " event=daily-report phase=skip day=" + day.getKey() + " reason=report-exists file=" + dailyReportFile.getAbsolutePath());
|
||||
deleteHourlyReports(day.getValue());
|
||||
log.info("runId=" + runId + " event=daily-report phase=skip day=" + day.getKey() + " reason=report-exists deletedLeftoverHourly=" + day.getValue().size() + " file=" + dailyReportFile.getAbsolutePath());
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
@@ -379,16 +381,15 @@ public class LogReportService {
|
||||
writeReport(dailyReportFile, dailyReportDocument(day.getKey(), day.getValue().size(), report));
|
||||
writtenReports.add(dailyReportFile);
|
||||
log.info("runId=" + runId + " event=daily-report phase=write result=success day=" + day.getKey() + " sourceReports=" + day.getValue().size() + " outputChars=" + report.length() + " file=" + dailyReportFile.getAbsolutePath());
|
||||
if (this.sb.getConfigBool(CONFIG_DELETE_HOURLY_AFTER_DAILY, false)) {
|
||||
deleteHourlyReports(day.getValue());
|
||||
log.info("runId=" + runId + " event=daily-report phase=delete-hourly day=" + day.getKey() + " sourceReports=" + day.getValue().size());
|
||||
}
|
||||
// the hourly reports are now bundled into the daily report and must be deleted
|
||||
deleteHourlyReports(day.getValue());
|
||||
log.info("runId=" + runId + " event=daily-report phase=delete-hourly day=" + day.getKey() + " sourceReports=" + day.getValue().size());
|
||||
} catch (final IOException e) {
|
||||
failedReports++;
|
||||
log.warn("runId=" + runId + " event=daily-report phase=write result=failure day=" + day.getKey() + " file=" + LogRedaction.redact(dailyReportFile.getAbsolutePath()) + " errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e));
|
||||
}
|
||||
}
|
||||
log.info("runId=" + runId + " event=daily-reports phase=end result=success completeDays=" + completeDays.size() + " written=" + writtenReports.size() + " skippedExisting=" + skippedExisting + " failed=" + failedReports + " durationMs=" + elapsed(start));
|
||||
log.info("runId=" + runId + " event=daily-reports phase=end result=success pastDays=" + pastDays.size() + " written=" + writtenReports.size() + " skippedExisting=" + skippedExisting + " failed=" + failedReports + " durationMs=" + elapsed(start));
|
||||
return writtenReports;
|
||||
}
|
||||
|
||||
@@ -573,16 +574,25 @@ public class LogReportService {
|
||||
return text.toString();
|
||||
}
|
||||
|
||||
private static Map<LocalDate, List<File>> completeHourlyReportSets(final File reportDirectory) {
|
||||
/**
|
||||
* Group all hourly reports of past days (strictly before today) by day, ordered
|
||||
* chronologically within each day. Every past day that still has hourly reports is
|
||||
* a compression candidate, no matter how many hours were actually reported - the
|
||||
* peer may have been offline for parts of the day. Only the current day is excluded
|
||||
* because its hourly reports are still accumulating.
|
||||
*/
|
||||
private static Map<LocalDate, List<File>> pastDayHourlyReportSets(final File reportDirectory) {
|
||||
final File[] files = reportDirectory.listFiles();
|
||||
if (files == null || files.length == 0) return Collections.emptyMap();
|
||||
|
||||
final LocalDate today = LocalDate.now();
|
||||
final Map<LocalDate, List<File>> candidates = new TreeMap<>();
|
||||
for (final File file : files) {
|
||||
if (!file.isFile()) continue;
|
||||
final LocalDateTime hour = parseHourlyReportFilename(file.getName());
|
||||
if (hour == null) continue;
|
||||
final LocalDate day = hour.toLocalDate();
|
||||
if (!day.isBefore(today)) continue;
|
||||
List<File> dayFiles = candidates.get(day);
|
||||
if (dayFiles == null) {
|
||||
dayFiles = new ArrayList<>();
|
||||
@@ -590,14 +600,11 @@ public class LogReportService {
|
||||
}
|
||||
dayFiles.add(file);
|
||||
}
|
||||
|
||||
final Map<LocalDate, List<File>> complete = new TreeMap<>();
|
||||
for (final Map.Entry<LocalDate, List<File>> entry : candidates.entrySet()) {
|
||||
if (entry.getValue().size() != 24) continue;
|
||||
final List<File> ordered = hourlyFilesForDay(reportDirectory, entry.getKey());
|
||||
if (ordered.size() == 24) complete.put(entry.getKey(), ordered);
|
||||
// the hourly filename pattern sorts chronologically
|
||||
for (final List<File> dayFiles : candidates.values()) {
|
||||
dayFiles.sort((a, b) -> a.getName().compareTo(b.getName()));
|
||||
}
|
||||
return complete;
|
||||
return candidates;
|
||||
}
|
||||
|
||||
private static LocalDateTime parseHourlyReportFilename(final String filename) {
|
||||
@@ -644,16 +651,6 @@ public class LogReportService {
|
||||
return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private static List<File> hourlyFilesForDay(final File reportDirectory, final LocalDate day) {
|
||||
final List<File> files = new ArrayList<>(24);
|
||||
for (int hour = 0; hour < 24; hour++) {
|
||||
final File file = new File(reportDirectory, hourlyReportFilename(day.atTime(hour, 0)));
|
||||
if (!file.isFile()) return Collections.emptyList();
|
||||
files.add(file);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
private static String dailyPrompt(final LocalDate day, final List<File> hourlyReports) throws IOException {
|
||||
final StringBuilder prompt = new StringBuilder(4096);
|
||||
prompt.append("Create one consolidated YaCy self-enhancement report for ")
|
||||
|
||||
@@ -30,9 +30,8 @@ public class LogReports_p {
|
||||
final LogReportService service = new LogReportService(sb);
|
||||
final File reportDirectory = service.getReportDirectory();
|
||||
|
||||
final int configuredMax = sb.getConfigInt(LogReportService.CONFIG_FEED_MAX_ENTRIES, LogReportService.DEFAULT_FEED_MAX_ENTRIES);
|
||||
final int requestedMax = post == null ? Math.min(20, configuredMax) : post.getInt("count", Math.min(20, configuredMax));
|
||||
final int maxEntries = Math.max(0, Math.min(requestedMax, configuredMax));
|
||||
// fixed display limit; the navigation column shows at most this many reports
|
||||
final int maxEntries = 100;
|
||||
|
||||
sb.setConfig("ui.LogReports_p.visited", "true");
|
||||
|
||||
@@ -66,25 +65,47 @@ public class LogReports_p {
|
||||
final List<ReportEntry> reports = service.discoverReports(maxEntries);
|
||||
|
||||
prop.put("modelConfigured", LogReportService.hasConfiguredLogReportModel() ? "1" : "0");
|
||||
prop.putNum("count", maxEntries);
|
||||
prop.putNum("maxcount", configuredMax);
|
||||
prop.putHTML("reportdir", reportDirectory.getAbsolutePath());
|
||||
prop.put("reportdirExists", reportDirectory.isDirectory() ? "1" : "0");
|
||||
prop.put("reportdirMissing", reportDirectory.isDirectory() ? "0" : "1");
|
||||
prop.put("jsonFeed", "api/logreports.json?count=" + maxEntries);
|
||||
prop.put("rssFeed", "api/logreports.rss?count=" + maxEntries);
|
||||
|
||||
// the report to show in the main view is selected by filename from the navigation
|
||||
// column; matching against the discovered list only prevents path traversal.
|
||||
// default is the newest report (first entry, list is sorted newest first)
|
||||
final String requestedReport = post == null ? "" : post.get("report", "");
|
||||
ReportEntry selected = null;
|
||||
for (final ReportEntry report : reports) {
|
||||
if (report.filename.equals(requestedReport)) {
|
||||
selected = report;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (selected == null && !reports.isEmpty()) selected = reports.get(0);
|
||||
|
||||
// navigation list and selected report live inside the #(hasReports)# alternative,
|
||||
// therefore all template keys carry the "hasReports_" prefix
|
||||
for (int i = 0; i < reports.size(); i++) {
|
||||
final ReportEntry report = reports.get(i);
|
||||
prop.putHTML("reports_" + i + "_filename", report.filename);
|
||||
prop.putHTML("reports_" + i + "_title", report.title);
|
||||
prop.putHTML("reports_" + i + "_published", report.published.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME));
|
||||
prop.putHTML("reports_" + i + "_type", report.daily ? "daily" : "hourly");
|
||||
prop.put("reports_" + i + "_daily", report.daily ? "1" : "0");
|
||||
prop.putNum("reports_" + i + "_chars", report.content.length());
|
||||
prop.putHTML("reports_" + i + "_content", report.content);
|
||||
final String p = "hasReports_reports_" + i + "_";
|
||||
prop.putHTML(p + "filename", report.filename);
|
||||
prop.putHTML(p + "title", report.title);
|
||||
prop.putHTML(p + "date", report.published.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")));
|
||||
prop.putHTML(p + "type", report.daily ? "daily" : "hourly");
|
||||
prop.put(p + "selected", report == selected ? "1" : "0");
|
||||
}
|
||||
prop.put("hasReports_reports", reports.size());
|
||||
|
||||
prop.put("hasReports_selectedReport", selected == null ? "0" : "1");
|
||||
if (selected != null) {
|
||||
prop.putHTML("hasReports_selectedReport_filename", selected.filename);
|
||||
prop.putHTML("hasReports_selectedReport_title", selected.title);
|
||||
prop.putHTML("hasReports_selectedReport_published", selected.published.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME));
|
||||
prop.putHTML("hasReports_selectedReport_type", selected.daily ? "daily" : "hourly");
|
||||
prop.putNum("hasReports_selectedReport_chars", selected.content.length());
|
||||
prop.putHTML("hasReports_selectedReport_content", selected.content);
|
||||
}
|
||||
prop.put("reports", reports.size());
|
||||
prop.putNum("reportCount", reports.size());
|
||||
prop.put("hasReports", reports.isEmpty() ? "0" : "1");
|
||||
prop.put("noReports", reports.isEmpty() ? "1" : "0");
|
||||
|
||||
Reference in new issue
Block a user