diff --git a/luci-app-fwlive/Makefile b/luci-app-fwlive/Makefile index 3cc315e73..154ddfbc4 100644 --- a/luci-app-fwlive/Makefile +++ b/luci-app-fwlive/Makefile @@ -11,8 +11,8 @@ LUCI_DESCRIPTION:=Live firewall log table (firewall4/nft; iptables-tagged lines LUCI_DEPENDS:=+luci-base +logd +jsonfilter LUCI_PKGARCH:=all -PKG_VERSION:=0.1.52 -PKG_RELEASE:=66 +PKG_VERSION:=0.1.53 +PKG_RELEASE:=67 # Reproducible build: honor SOURCE_DATE_EPOCH from the build environment (set by docker-sdk.sh / CI). ifdef SOURCE_DATE_EPOCH diff --git a/luci-app-fwlive/htdocs/luci-static/resources/fwlive/constants.js b/luci-app-fwlive/htdocs/luci-static/resources/fwlive/constants.js index f4a3b0f73..a342afe3c 100644 --- a/luci-app-fwlive/htdocs/luci-static/resources/fwlive/constants.js +++ b/luci-app-fwlive/htdocs/luci-static/resources/fwlive/constants.js @@ -9,7 +9,7 @@ */ return baseclass.extend({ /* Keep in sync with openwrt-feed/luci-app-fwlive/Makefile PKG_VERSION. */ - APP_VERSION: '0.1.52', + APP_VERSION: '0.1.53', ROW_LIMIT_OPTIONS: [25, 50, 100, 250, 500, 1000, 2000], DEFAULT_ROW_LIMIT: 100, FETCH_MODE_OPTIONS: ['auto', 'manual'], diff --git a/luci-app-trafficctl/Makefile b/luci-app-trafficctl/Makefile index a19704fc7..21c4f039d 100644 --- a/luci-app-trafficctl/Makefile +++ b/luci-app-trafficctl/Makefile @@ -2,7 +2,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=luci-app-trafficctl PKG_VERSION:=1.21.6 -PKG_RELEASE:=10 +PKG_RELEASE:=11 PKG_LICENSE:=Apache-2.0 PKG_MAINTAINER:=Denis Iusupov diff --git a/luci-app-trafficctl/htdocs/luci-static/resources/view/trafficctl/history.js b/luci-app-trafficctl/htdocs/luci-static/resources/view/trafficctl/history.js new file mode 100644 index 000000000..79c68b559 --- /dev/null +++ b/luci-app-trafficctl/htdocs/luci-static/resources/view/trafficctl/history.js @@ -0,0 +1,474 @@ +'use strict'; +'require view'; +'require rpc'; +'require ui'; + +(function() { + if (!document.querySelector('link[href*="trafficctl/status.css"]')) { + var lnk = document.createElement('link'); + lnk.rel = 'stylesheet'; + lnk.type = 'text/css'; + lnk.href = '/luci-static/resources/view/trafficctl/status.css'; + document.head.appendChild(lnk); + } +})(); + +var callHistoryStatus = rpc.declare({ + object: 'luci.trafficctl', + method: 'history_status', + expect: { result: {} } +}); + +var callHistoryPeriods = rpc.declare({ + object: 'luci.trafficctl', + method: 'history_periods', + expect: { result: [] } +}); + +var callHistoryRead = rpc.declare({ + object: 'luci.trafficctl', + method: 'history_read', + params: ['period', 'to'], + expect: { result: [] } +}); + +var callHistoryFlush = rpc.declare({ + object: 'luci.trafficctl', + method: 'history_flush', + expect: { result: {} } +}); + +var callHistoryConfigSet = rpc.declare({ + object: 'luci.trafficctl', + method: 'history_config_set', + params: ['enabled', 'dir', 'period', 'interval', 'sample_interval', 'keep'] +}); + +function fmtBytes(b) { + b = b || 0; + if (b >= 1000000000000) return (b / 1000000000000).toFixed(2) + ' TB'; + if (b >= 1000000000) return (b / 1000000000).toFixed(b >= 10000000000 ? 1 : 2) + ' GB'; + if (b >= 1000000) return (b / 1000000).toFixed(b >= 10000000 ? 0 : 1) + ' MB'; + if (b >= 1000) return (b / 1000).toFixed(0) + ' KB'; + return b + ' B'; +} + +function fmtAgo(ts) { + if (!ts) return '—'; + var d = Math.floor(Date.now() / 1000) - ts; + if (d < 0) d = 0; + if (d < 90) return _('just now'); + if (d < 5400) return Math.round(d / 60) + ' ' + _('min ago'); + if (d < 172800) return Math.round(d / 3600) + ' ' + _('h ago'); + return Math.round(d / 86400) + ' ' + _('days ago'); +} + +// The three verdicts trafficctl-history.sh can report, and what each one means +// for a number on this page. Keyed on exactly the strings hist_accuracy emits; +// an unknown value must fall through to a warning rather than to silence, +// because the dangerous direction is presenting an unverified number as a +// measurement. +var ACCURACY = { + 'counter': { + cls: 'tc-c-ok', + icon: '●', + text: _('Exact. Byte counts come from cumulative nftables counters, so they do not depend on how often the sampler runs. The TCP/UDP split is unavailable from this source.') + }, + 'sampled': { + cls: 'tc-c-muted', + icon: '○', + text: _('Sampled. Byte counts come from conntrack, which only reports flows that are alive when it is read — a connection that both starts and finishes between two samples is not counted. Shorten the sampling interval to narrow that gap.') + }, + 'degraded': { + cls: 'tc-c-warn', + icon: '⚠', + text: _('Lower bound, not a measurement. Flow offloading is active without counter support, so the kernel counters this history is built from stop moving while traffic keeps flowing. The real figures are higher by an unknown amount — use these only as a floor.') + } +}; + +function accuracyOf(verdict) { + return ACCURACY[verdict] || { + cls: 'tc-c-warn', + icon: '⚠', + text: _('The accuracy of these figures could not be determined.') + }; +} + +return view.extend({ + st: {}, + periods: [], + rows: [], + sel: '', + selTo: '', + sortKey: 'total', + tbody: null, + statusEl: null, + bannerEl: null, + + handleSaveApply: null, + handleSave: null, + handleReset: null, + + load: function() { + return Promise.all([ + callHistoryStatus().catch(function() { return {}; }), + callHistoryPeriods().catch(function() { return []; }) + ]); + }, + + totalOf: function(r) { + return (r.bytes_in || 0) + (r.bytes_out || 0); + }, + + sorted: function() { + var self = this; + var rows = this.rows.slice(); + rows.sort(function(a, b) { + if (self.sortKey === 'name') { + var an = (a.name || a.ip || '').toLowerCase(); + var bn = (b.name || b.ip || '').toLowerCase(); + return an < bn ? -1 : (an > bn ? 1 : 0); + } + if (self.sortKey === 'last') return (b.last_seen || 0) - (a.last_seen || 0); + if (self.sortKey === 'down') return (b.bytes_in || 0) - (a.bytes_in || 0); + if (self.sortKey === 'up') return (b.bytes_out || 0) - (a.bytes_out || 0); + return self.totalOf(b) - self.totalOf(a); + }); + return rows; + }, + + renderRow: function(r) { + var label = r.name || r.ip || r.key; + + // An ip: key means no MAC could be established for this device — a + // client behind a downstream router, or one in extra_subnets. Its + // history is only as stable as its address, and saying so is the + // point: the alternative was guessing, which merges devices. + var ident = []; + if (r.key_kind === 'mac' && r.mac && r.mac !== '-') { + ident.push(E('span', { 'class': 'tc-mono' }, r.mac)); + } else { + ident.push(E('span', { + 'class': 'tc-c-warn', + 'title': _('No MAC address is known for this device, so its history is keyed on its IP address. If that address is reassigned, earlier traffic recorded here may belong to a different device.') + }, _('IP-keyed'))); + } + if (r.ip) { + ident.push(document.createTextNode(' ')); + ident.push(E('span', { 'class': 'tc-c-faint tc-mono' }, r.ip)); + } + + var nameCell = [ + E('div', { 'class': 'tc-fw-bold' }, label), + E('div', { 'class': 'tc-c-faint', 'style': 'font-size:10px' }, ident) + ]; + if (r.degraded) { + nameCell.push(E('div', { + 'class': 'tc-c-warn', + 'style': 'font-size:10px', + 'title': _('While this total was being accumulated, flow offloading froze the counters it is built from. The figure is a floor, not a measurement.') + }, '⚠ ' + _('undercounted'))); + } + + return E('tr', { 'class': 'tr' }, [ + E('td', { 'class': 'td' }, nameCell), + E('td', { 'class': 'td tc-right tc-mono tc-sm' }, fmtBytes(r.bytes_in)), + E('td', { 'class': 'td tc-right tc-mono tc-sm' }, fmtBytes(r.bytes_out)), + E('td', { 'class': 'td tc-right tc-mono tc-fw-bold' }, fmtBytes(this.totalOf(r))), + E('td', { 'class': 'td tc-center tc-sm tc-c-muted' }, fmtAgo(r.last_seen)), + E('td', { 'class': 'td tc-center tc-sm tc-c-faint' }, + (r.periods || []).join(', ')) + ]); + }, + + renderTable: function() { + var self = this; + while (this.tbody.firstChild) this.tbody.removeChild(this.tbody.firstChild); + + if (!this.st.enabled) { + this.tbody.appendChild(E('tr', { 'class': 'tr' }, [ + E('td', { 'class': 'td tc-c-faint', 'colspan': 6 }, + _('Traffic history is switched off. Nothing is being recorded.')) + ])); + return; + } + if (!this.rows.length) { + this.tbody.appendChild(E('tr', { 'class': 'tr' }, [ + E('td', { 'class': 'td tc-c-faint', 'colspan': 6 }, + _('No traffic has been recorded for this period yet. The first figures appear after the next flush.')) + ])); + return; + } + + var rows = this.sorted(); + var tin = 0, tout = 0; + rows.forEach(function(r) { + tin += (r.bytes_in || 0); + tout += (r.bytes_out || 0); + self.tbody.appendChild(self.renderRow(r)); + }); + + // A total over rows that include a degraded one is itself a floor, so + // it has to carry the same caveat rather than look authoritative. + var anyDegraded = rows.some(function(r) { return !!r.degraded; }); + this.tbody.appendChild(E('tr', { 'class': 'tr', 'style': 'border-top:2px solid var(--border-color-medium,#ccc)' }, [ + E('td', { 'class': 'td tc-fw-bold' }, [ + document.createTextNode(_('Total') + ' (' + rows.length + ' ' + _('devices') + ')'), + anyDegraded ? E('span', { + 'class': 'tc-c-warn', + 'style': 'margin-left:6px;font-size:10px' + }, '⚠ ' + _('at least')) : '' + ]), + E('td', { 'class': 'td tc-right tc-mono tc-sm tc-fw-bold' }, fmtBytes(tin)), + E('td', { 'class': 'td tc-right tc-mono tc-sm tc-fw-bold' }, fmtBytes(tout)), + E('td', { 'class': 'td tc-right tc-mono tc-fw-bold' }, fmtBytes(tin + tout)), + E('td', { 'class': 'td' }, ''), + E('td', { 'class': 'td' }, '') + ])); + }, + + loadPeriod: function() { + var self = this; + self.statusEl.textContent = _('Loading…'); + return callHistoryRead(self.sel || '', self.selTo || '').then(function(rows) { + self.rows = rows || []; + self.statusEl.textContent = ''; + self.renderTable(); + }).catch(function(e) { + self.statusEl.textContent = ''; + ui.addNotification(null, E('p', e.message), 'error'); + }); + }, + + refreshStatus: function() { + var self = this; + return Promise.all([callHistoryStatus(), callHistoryPeriods()]) + .then(function(res) { + self.st = res[0] || {}; + self.periods = res[1] || []; + self.renderBanner(); + }).catch(function() {}); + }, + + renderBanner: function() { + var a = accuracyOf(this.st.accuracy); + while (this.bannerEl.firstChild) this.bannerEl.removeChild(this.bannerEl.firstChild); + + if (!this.st.enabled) { + this.bannerEl.appendChild(E('div', { 'class': 'tc-c-muted' }, [ + E('span', { 'class': 'tc-fw-bold' }, _('Off.') + ' '), + document.createTextNode(_('History is not recorded by default: it writes to the router\'s flash on a schedule, and every other persistent store in this app was deliberately kept off that path. Enable it below.')) + ])); + return; + } + + var notes = [ + E('div', { 'class': a.cls }, [ + E('span', { 'class': 'tc-fw-bold' }, a.icon + ' '), + document.createTextNode(a.text) + ]) + ]; + + if (this.st.error) { + notes.push(E('div', { 'class': 'tc-c-warn tc-fw-bold', 'style': 'margin-top:4px' }, + '⚠ ' + _('Recording is paused: the last write failed.') + ' ' + this.st.error)); + } else if (!this.st.writable) { + notes.push(E('div', { 'class': 'tc-c-warn', 'style': 'margin-top:4px' }, + '⚠ ' + _('The store directory is not writable yet.') + ' ' + (this.st.dir || ''))); + } + + notes.push(E('div', { 'class': 'tc-c-faint', 'style': 'margin-top:4px;font-size:11px' }, + _('Store') + ': ' + (this.st.dir || '—') + + ' · ' + _('period') + ': ' + (this.st.period_kind || '—') + + ' · ' + _('flush every') + ' ' + (this.st.interval || 0) + ' s' + + ' · ' + _('sampled every') + ' ' + (this.st.sample_interval || 0) + ' s' + + ' · ' + _('keeping') + ' ' + (this.st.keep || 0) + ' ' + _('periods') + + ' · ' + _('last flush') + ': ' + fmtAgo(this.st.last_flush))); + + notes.forEach(function(n) { this.bannerEl.appendChild(n); }, this); + }, + + exportCsv: function() { + var self = this; + var head = ['key', 'key_kind', 'mac', 'ip', 'name', 'bytes_in', 'bytes_out', + 'bytes_total', 'first_seen', 'last_seen', 'degraded', 'periods']; + var lines = [head.join(',')]; + this.sorted().forEach(function(r) { + // Quote and escape every field: a device name comes from a DHCP + // lease or a PTR record and can contain a comma or a quote. + var vals = [r.key, r.key_kind, r.mac, r.ip, r.name || '', + r.bytes_in, r.bytes_out, self.totalOf(r), + r.first_seen, r.last_seen, r.degraded ? 'true' : 'false', + (r.periods || []).join(' ')]; + lines.push(vals.map(function(v) { + return '"' + String(v === undefined || v === null ? '' : v).replace(/"/g, '""') + '"'; + }).join(',')); + }); + var blob = new Blob([lines.join('\n') + '\n'], { type: 'text/csv' }); + var url = URL.createObjectURL(blob); + var a = E('a', { + 'href': url, + 'download': 'trafficctl-history-' + (this.sel || this.st.period || 'current') + '.csv' + }); + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + }, + + render: function(data) { + var self = this; + this.st = data[0] || {}; + this.periods = data[1] || []; + this.sel = this.st.period || ''; + this.selTo = ''; + + this.statusEl = E('span', { 'class': 'tc-c-muted', 'style': 'margin-left:10px;font-size:12px' }); + this.bannerEl = E('div', { 'style': 'margin:6px 0 10px 0;font-size:12px' }); + + // ── period picker ─────────────────────────────────────────────────── + var picker = E('div', { 'style': 'margin-bottom:8px' }); + var known = this.periods.map(function(p) { return p.period; }); + if (this.st.period && known.indexOf(this.st.period) < 0) known.unshift(this.st.period); + known.sort().reverse(); + + function periodButton(label, from, to, title) { + var active = (self.sel === from && self.selTo === to); + var b = E('button', { + 'class': 'cbi-button ' + (active ? 'cbi-button-apply' : ''), + 'style': 'font-size:11px;padding:1px 8px;margin-right:4px', + 'title': title || '' + }, label); + b.addEventListener('click', function() { + self.sel = from; + self.selTo = to; + self.loadPeriod(); + // Re-render the picker so the active state follows the click. + var fresh = buildPicker(); + picker.parentNode.replaceChild(fresh, picker); + picker = fresh; + }); + return b; + } + + function buildPicker() { + var p = E('div', { 'style': 'margin-bottom:8px' }); + p.appendChild(E('span', { 'class': 'tc-c-faint', 'style': 'margin-right:6px;font-size:11px' }, + _('Period') + ':')); + known.forEach(function(name) { + p.appendChild(periodButton(name, name, '', _('Show only this period'))); + }); + if (known.length > 1) { + p.appendChild(periodButton(_('All'), known[known.length - 1], known[0], + _('Aggregate every stored period'))); + } + picker = p; + return p; + } + picker = buildPicker(); + + // ── table ─────────────────────────────────────────────────────────── + this.tbody = E('tbody', {}); + + function sortHeader(label, key, cls, title) { + var th = E('th', { + 'class': 'th ' + (cls || ''), + 'style': 'cursor:pointer', + 'title': title || _('Sort by this column') + }, label + (self.sortKey === key ? ' ▾' : '')); + th.addEventListener('click', function() { + self.sortKey = key; + self.renderTable(); + var fresh = buildHead(); + thead.parentNode.replaceChild(fresh, thead); + thead = fresh; + }); + return th; + } + + var thead; + function buildHead() { + thead = E('thead', {}, [ + E('tr', { 'class': 'tr cbi-section-table-titles' }, [ + sortHeader(_('Device'), 'name'), + sortHeader(_('Download'), 'down', 'tc-right'), + sortHeader(_('Upload'), 'up', 'tc-right'), + sortHeader(_('Total'), 'total', 'tc-right'), + sortHeader(_('Last seen'), 'last', 'tc-center', + _('When this device last moved any traffic, not when it was last connected')), + E('th', { 'class': 'th tc-center' }, _('Periods')) + ]) + ]); + return thead; + } + buildHead(); + + var table = E('table', { 'class': 'table tc-table' }, [thead, this.tbody]); + + this.renderBanner(); + this.renderTable(); + if (this.st.enabled) this.loadPeriod(); + + // ── controls ──────────────────────────────────────────────────────── + var enableBtn = E('button', { 'class': 'cbi-button cbi-button-apply' }, + this.st.enabled ? _('Switch off') : _('Switch on')); + enableBtn.addEventListener('click', function() { + enableBtn.disabled = true; + var want = !self.st.enabled; + callHistoryConfigSet(want, null, null, null, null, null).then(function(res) { + enableBtn.disabled = false; + if (res && res.ok === false) { + ui.addNotification(null, E('p', res.msg || _('Operation failed')), 'error'); + return; + } + return self.refreshStatus().then(function() { + enableBtn.textContent = self.st.enabled ? _('Switch off') : _('Switch on'); + if (self.st.enabled) return self.loadPeriod(); + self.rows = []; + self.renderTable(); + }); + }).catch(function(e) { + enableBtn.disabled = false; + ui.addNotification(null, E('p', e.message), 'error'); + }); + }); + + var flushBtn = E('button', { + 'class': 'cbi-button', + 'style': 'margin-left:6px', + 'title': _('Write the figures accumulated so far to the store now, without waiting for the next scheduled flush') + }, _('Flush now')); + flushBtn.addEventListener('click', function() { + flushBtn.disabled = true; + self.statusEl.textContent = _('Flushing…'); + callHistoryFlush().then(function(res) { + flushBtn.disabled = false; + self.statusEl.textContent = ''; + if (res && res.ok === false) { + ui.addNotification(null, E('p', res.reason || _('Flush failed')), 'error'); + } + return self.refreshStatus().then(function() { return self.loadPeriod(); }); + }).catch(function(e) { + flushBtn.disabled = false; + self.statusEl.textContent = ''; + ui.addNotification(null, E('p', e.message), 'error'); + }); + }); + + var csvBtn = E('button', { 'class': 'cbi-button', 'style': 'margin-left:6px' }, + _('Export CSV')); + csvBtn.addEventListener('click', function() { self.exportCsv(); }); + + return E('div', {}, [ + E('h2', {}, _('Traffic History')), + E('div', { 'class': 'cbi-map-descr' }, [ + document.createTextNode(_('Per-device totals accumulated per accounting period, including devices that are not connected now. Figures survive reboots; the live view on the Devices tab does not.')), + this.statusEl + ]), + this.bannerEl, + E('div', { 'style': 'margin-bottom:8px' }, [enableBtn, flushBtn, csvBtn]), + picker, + table + ]); + } +}); diff --git a/luci-app-trafficctl/htdocs/luci-static/resources/view/trafficctl/status.js b/luci-app-trafficctl/htdocs/luci-static/resources/view/trafficctl/status.js index 409b5c715..407360435 100644 --- a/luci-app-trafficctl/htdocs/luci-static/resources/view/trafficctl/status.js +++ b/luci-app-trafficctl/htdocs/luci-static/resources/view/trafficctl/status.js @@ -4390,7 +4390,32 @@ return view.extend({ 'a router with modern OpenWrt and fw4 support will have this working out of the box.')), docLinks, ]); - } else { + } else if (mode === 'software-counter') { + // Split out from plain 'software' (#77). Both end up with + // accurate graphs, which is why this was cosmetic rather than + // misleading — but it is a different mechanism, and the + // counter flag is exactly the detail someone opens this banner + // to understand. Saying "measured via nftables counters" here + // describes a path that is NOT producing the numbers on this + // router: with the flag set, the flowtable syncs offloaded + // byte counts back to conntrack and tctl_pick_byte_source + // stays on conntrack. + bg = 'rgba(243,156,18,0.10)'; + border = '#f39c12'; + icon = 'ℹ️'; + body = E('div', {}, [ + para(bold(_('Software flow offloading is active, with counter sync.'))), + para(_('The kernel fast-paths established connections through a flowtable for ' + + 'higher throughput.')), + para([_('The flowtable carries the '), + E('code', {}, 'counter'), + _(' flag (Linux 5.7+, set automatically by fw4), so offloaded byte counts are ' + + 'synced back to conntrack. Speed and totals come from conntrack as usual and ' + + 'are accurate — the nftables counter fallback is not needed on this router.')]), + para(_('Traffic shaping and blocking are not affected.')), + docLinks, + ]); + } else if (mode === 'software') { bg = 'rgba(243,156,18,0.10)'; border = '#f39c12'; icon = 'ℹ️'; @@ -4403,6 +4428,24 @@ return view.extend({ para(_('Traffic shaping and blocking are not affected.')), docLinks, ]); + } else { + // A mode tctl_get_offload_mode learned and this panel has not. + // Deliberately vague rather than inheriting whichever branch + // happened to be last: #77 was exactly that, software-counter + // falling into the plain-software text and describing a + // mechanism the router was not using. test_offload_ui.sh fails + // when a backend mode has no branch here, so reaching this is + // a bug rather than a state to word carefully for. + bg = 'rgba(243,156,18,0.10)'; + border = '#f39c12'; + icon = 'ℹ️'; + body = E('div', {}, [ + para(bold(_('Flow offloading is active.'))), + para(_('This build of trafficctl does not have a description for the mode the ' + + 'router reported, so how it affects monitoring is not stated here rather ' + + 'than guessed at.')), + docLinks, + ]); } var banner = E('div', { diff --git a/luci-app-trafficctl/root/etc/config/trafficctl b/luci-app-trafficctl/root/etc/config/trafficctl index ff30987d0..1d707bf5d 100644 --- a/luci-app-trafficctl/root/etc/config/trafficctl +++ b/luci-app-trafficctl/root/etc/config/trafficctl @@ -72,6 +72,31 @@ config cut 'cut' # thing standing between the operator and a permanent lockout. option default_duration '0' +config history 'history' + # Durable per-device traffic history (#89, #90). OFF by default: every + # other persistent store in this app was deliberately kept off the write + # path, and this one writes to flash on a schedule. + option enabled '0' + # Where the period files live. One file per accounting period, replaced + # atomically through a temp file in this same directory — so a network + # mount works, but see the note on `interval` before reaching for one. + option dir '/etc/trafficctl/history' + # day | week | month. The period IS the reset: a new period is a new file. + option period 'month' + # Seconds between flushes to durable storage. Floored at 300. Hourly is + # roughly 24 writes a day of a few KB, which is not a wear problem — this + # is the setting to raise if flash wear is the worry, in preference to + # pointing `dir` at a share that can hang when the NAS sleeps. + option interval '3600' + # Seconds between samples of the live accumulator. tmpfs only, no flash. + # Separate from `interval` on purpose and much shorter: with a conntrack + # byte source, a flow that starts and ends between two samples is + # invisible to both, so an hourly sample would miss most short traffic. + option sample_interval '60' + # How many FINISHED periods to retain. The current one is never a + # retention candidate. + option keep '12' + config logging 'logging' option enabled '1' option log_file '/tmp/trafficctl/activity.log' diff --git a/luci-app-trafficctl/root/etc/init.d/trafficctl-history b/luci-app-trafficctl/root/etc/init.d/trafficctl-history new file mode 100755 index 000000000..15c7dc723 --- /dev/null +++ b/luci-app-trafficctl/root/etc/init.d/trafficctl-history @@ -0,0 +1,41 @@ +#!/bin/sh /etc/rc.common +# shellcheck shell=dash +# Sampler and flusher for the durable per-period traffic history. +# +# Why a service and not a cron line: there are TWO cadences and they are not +# interchangeable. Sampling must be frequent — with a conntrack source, a flow +# that starts and finishes between two samples is invisible to all of them — +# and costs a tmpfs write. Flushing touches durable storage and is therefore +# rare. One process runs both; see trafficctl-history.sh for the accuracy +# model that forces the split. +# +# Respawns, unlike the cut keeper: this is a steady-state service, and a +# sampler that quietly died would not fail loudly — it would just leave a +# month with a plausible-looking number in it that is far too low. +# +# Enabled at boot, but the loop itself refuses to do anything while +# trafficctl.history.enabled is 0, and re-reads that on every tick. So +# switching history on in the UI does not require knowing there is a service +# behind it. +# +# START/STOP/USE_PROCD are consumed by the sourced /etc/rc.common framework. +# shellcheck disable=SC2034 +START=96 +STOP=10 +USE_PROCD=1 + +start_service() { + procd_open_instance history + procd_set_param command /usr/local/bin/trafficctl-history.sh keeper + procd_set_param respawn 3600 5 0 + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_close_instance +} + +service_triggers() { + # A change to the store location or the intervals is picked up by the loop + # itself, but enabling history for the first time should start sampling + # without waiting for a reboot. + procd_add_reload_trigger "trafficctl" +} diff --git a/luci-app-trafficctl/root/usr/libexec/rpcd/luci.trafficctl b/luci-app-trafficctl/root/usr/libexec/rpcd/luci.trafficctl index 1c1e13afe..54e89b7cb 100755 --- a/luci-app-trafficctl/root/usr/libexec/rpcd/luci.trafficctl +++ b/luci-app-trafficctl/root/usr/libexec/rpcd/luci.trafficctl @@ -71,6 +71,12 @@ list) printf '"cut_status":{},' printf '"cut_set":{"active":"bool","duration":"int","persist":"bool"},' printf '"activity_log":{"lines":"int"},' + printf '"history_status":{},' + printf '"history_read":{"period":"str","to":"str"},' + printf '"history_periods":{},' + printf '"history_config_get":{},' + printf '"history_config_set":{"enabled":"bool","dir":"str","period":"str","interval":"int","sample_interval":"int","keep":"int"},' + printf '"history_flush":{},' printf '"version":{}' printf '}' ;; @@ -596,6 +602,99 @@ call) echo '{"ok":true,"lines":[]}' fi ;; + history_status) + printf '{"result":' + $SCRIPTS/trafficctl-history.sh status + printf '}' + ;; + history_periods) + printf '{"result":' + $SCRIPTS/trafficctl-history.sh periods + printf '}' + ;; + history_read) + read -r input + # Period ids are compared as strings inside the script and never + # interpolated into a path, but they still come from the network, so + # the shape is pinned here: a value that is not a period id is + # rejected rather than passed through and silently matching nothing. + hp=$(echo "$input" | jsonfilter -e '@.period' 2>/dev/null) + ht=$(echo "$input" | jsonfilter -e '@.to' 2>/dev/null) + for v in "$hp" "$ht"; do + [ -z "$v" ] && continue + case "$v" in + [0-9][0-9][0-9][0-9]-[0-9][0-9]) ;; + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; + [0-9][0-9][0-9][0-9]-W[0-9][0-9]) ;; + *) echo '{"result":[]}'; exit 0 ;; + esac + done + printf '{"result":' + $SCRIPTS/trafficctl-history.sh read "$hp" "$ht" + printf '}' + ;; + history_config_get) + printf '{"result":' + $SCRIPTS/trafficctl-history.sh status + printf '}' + ;; + history_config_set) + read -r input + uci -q get trafficctl.history >/dev/null 2>&1 || uci set trafficctl.history=history + val=$(echo "$input" | jsonfilter -e '@.dir' 2>/dev/null) + if [ -n "$val" ]; then + # Validated by the script that owns the rule, not by a second copy + # of it here. The directory is both a mkdir -p and a delete target + # (prune removes *.tsv inside it), so an unconstrained value would + # hand this write method arbitrary directory creation and the + # removal of any .tsv on the filesystem. + if ! $SCRIPTS/trafficctl-history.sh validate-dir "$val" 2>/dev/null; then + echo '{"ok":false,"msg":"dir must be under /etc/trafficctl, /mnt, /media, /srv or /tmp"}' + exit 0 + fi + uci set trafficctl.history.dir="$val" + fi + val=$(echo "$input" | jsonfilter -e '@.period' 2>/dev/null) + if [ -n "$val" ]; then + case "$val" in + day|week|month) uci set trafficctl.history.period="$val" ;; + *) echo '{"ok":false,"msg":"period must be day, week or month"}'; exit 0 ;; + esac + fi + for k in interval sample_interval keep; do + val=$(echo "$input" | jsonfilter -e "@.$k" 2>/dev/null) + [ -z "$val" ] && continue + case "$val" in + ''|*[!0-9]*) echo "{\"ok\":false,\"msg\":\"$k must be a number\"}"; exit 0 ;; + esac + uci set "trafficctl.history.$k=$val" + done + val=$(echo "$input" | jsonfilter -e '@.enabled' 2>/dev/null) + if [ -n "$val" ]; then + uci set "trafficctl.history.enabled=$([ "$val" = "true" ] && echo 1 || echo 0)" + fi + tctl_commit + # The keeper re-reads the intervals and the directory on every tick, + # but it has to be RUNNING to do that — switching history on for the + # first time must not wait for a reboot. + if [ -x /etc/init.d/trafficctl-history ]; then + if [ "$(uci -q get trafficctl.history.enabled)" = "1" ]; then + /etc/init.d/trafficctl-history enable 2>/dev/null + /etc/init.d/trafficctl-history start 2>/dev/null + else + /etc/init.d/trafficctl-history stop 2>/dev/null + fi + fi + tctl_log "config_change" "history" "" "luci" "${TCTL_SRC:-local}" + echo '{"ok":true}' + ;; + history_flush) + # A "flush now" button. Writes to durable storage, so it is a write + # method even though it only persists numbers that already exist. + printf '{"result":' + $SCRIPTS/trafficctl-history.sh flush + printf '}' + ;; version) ver=$(opkg info luci-app-trafficctl 2>/dev/null | awk '/^Version:/{print $2}') [ -z "$ver" ] && ver="dev" diff --git a/luci-app-trafficctl/root/usr/local/bin/trafficctl-history.sh b/luci-app-trafficctl/root/usr/local/bin/trafficctl-history.sh new file mode 100755 index 000000000..f4646da90 --- /dev/null +++ b/luci-app-trafficctl/root/usr/local/bin/trafficctl-history.sh @@ -0,0 +1,753 @@ +#!/bin/sh +# shellcheck shell=dash +# Durable per-device traffic history, accumulated per accounting period. +# +# WHY THIS EXISTS (#89, #90) +# trafficctl-totals.sh already computes correct lifetime totals, but its store +# is /tmp/trafficctl_totals.state — tmpfs, deliberately: that file is +# rewritten on every LuCI poll (2-10 s) and every metrics scrape, and that +# write volume into flash would wear the device out. So the numbers existed +# and had nowhere durable to go. Everything here is persistence and +# windowing; no new accounting. +# +# ONE FILE PER PERIOD +# /.tsv, where period is 2026-10-09 / 2026-W41 / 2026-10. The +# "reset every day/week/month" request is then a no-op — a new period is a +# new filename — and retention is a file delete rather than a rewrite of +# everything. A finished period's file is never touched again, which also +# means a crash can only ever damage the current one. +# +# TSV, NOT JSON +# An interrupted write to a JSON file costs the whole file; to a TSV, its +# last line. This runs on a device that can lose power mid-write and whose +# store may sit on a network mount. Rewrites still go through a temp file in +# the SAME directory and a mv, because a temp file elsewhere plus a +# cross-filesystem mv is not atomic — and "elsewhere" is exactly what +# happens when dir points at a mount. +# +# DELTAS, NOT SNAPSHOTS +# A flush folds in only the growth since the previous flush, tracked in a +# tmpfs cursor. If the live total is BELOW the cursor the accumulator was +# reset (reboot, counter source switch), so the flush rebaselines and adds +# nothing rather than subtracting — a reboot costs at most one interval of +# traffic instead of corrupting the period. The cursor lives in tmpfs on +# purpose: it must die together with the accumulator it tracks, or the first +# flush after a reboot would subtract a stale high-water mark from a +# restarted counter. +# +# HOW ACCURATE THE STORED NUMBERS ARE +# Three different answers, and the store has to say which one applies rather +# than present all three as "bytes used". +# +# 1. UNCOUNTERED OFFLOAD — "degraded". With flow offloading on and no +# counter-flagged flowtable, an offloaded flow's counters stop moving +# while traffic continues; if the nft-map fallback is also unavailable, +# there is nothing left that sees those packets. The total is then a LOWER +# BOUND that can be arbitrarily far below reality, not an approximation +# around it. trafficctl-bytes.sh flags this, totals keeps the flag STICKY +# per device, and it is stored per row here — a consumer must label such a +# row, never add it to a trustworthy sum. +# +# 2. CONNTRACK SOURCE — sampled, so short flows can be missed. Conntrack +# reports what its CURRENTLY TRACKED flows have carried, which means a +# flow that starts and finishes entirely between two samples was never +# seen by any of them. At the dashboard's 2 s poll that is noise; at an +# hourly flush it would be most of the traffic. That is why `sample` is a +# subcommand of its own, driven at sample_interval (tmpfs only, no flash +# writes), and why the flush interval is NOT the sampling interval. +# +# 3. NFT COUNTER MAPS — cumulative since the table was built, so independent +# of sampling rate and exact. They cannot split TCP from UDP, which is why +# those columns report -1 rather than 0. +# +# Common to all three: the counters are read BEFORE the shaper, so they +# describe what arrived for a device, not what was delivered to it. +# +# NEVER IN A REQUEST PATH +# flush is driven by its own timer. A flash write triggered by somebody +# having a browser tab open is the wrong coupling, and if dir is on a dead +# NFS/CIFS mount the writer blocks — which must not be able to stall a device +# poll. Writes are wrapped in a timeout where one is available and a failure +# parks history in a visible error state instead of retrying silently. +# +# Usage: +# trafficctl-history.sh sample advance the live accumulator (tmpfs only) +# trafficctl-history.sh flush fold live totals into the current period +# trafficctl-history.sh read [period] JSON rows for one period (default: current) +# trafficctl-history.sh read JSON rows aggregated over a range +# trafficctl-history.sh periods JSON list of stored periods +# trafficctl-history.sh prune drop periods beyond the retention count +# trafficctl-history.sh status JSON: enabled, dir, writability, last error +# trafficctl-history.sh keeper the procd service loop (sample + flush) +# trafficctl-history.sh validate-dir

exit 0 if p is an allowed store dir + +. /usr/local/bin/trafficctl-fw.sh + +# Named by absolute path, as they must be on the router; the tests sed these +# lines into their scratch directory rather than have the shipped script carry +# test-only overrides (same approach as trafficctl-totals.sh and the ledger +# paths in trafficctl-fw.sh). All four are tmpfs: see DELTAS, NOT SNAPSHOTS +# above for why the cursor in particular must not survive a reboot. +CURSOR="/tmp/trafficctl_history.cursor" +ERRFILE="/tmp/trafficctl_history.err" +STAMP="/tmp/trafficctl_history.stamp" +LOCKD="/tmp/trafficctl_history.lock.d" +TOTALS_STATE="/tmp/trafficctl_totals.state" + +# Row layout, tab-separated: +# key mac ip name rx tx tcp udp first_seen last_seen degraded +# key is "mac:" or "ip:" (see hist_key). Fields are appended at the +# END in later versions so an older reader keeps working, same rule the +# shape/limit record formats follow. +FORMAT=1 + +hist_enabled() { + [ "$(uci -q get trafficctl.history.enabled 2>/dev/null)" = "1" ] +} + +# The store directory is both a mkdir -p target and a delete target (prune +# removes *.tsv inside it), so an unconstrained value would hand the write ACL +# arbitrary directory creation plus the removal of any .tsv on the filesystem. +# That is the same mistake tctl_validate_log_file exists to prevent, and it is +# enforced HERE as well as in the rpcd setter — a value that reached the config +# by another route must not be honoured either. +# +# The allowed roots cover the cases the feature is for: the package's own state +# directory, and the usual mount points, which is where a network share or a +# USB disk lands. +HIST_DIR_DEFAULT="/etc/trafficctl/history" + +hist_validate_dir() { + local d="$1" + case "$d" in + */../*|*/..|../*|..) return 1 ;; + *[\'\"\$\`\|\;\&\<\>]*) return 1 ;; + /etc/trafficctl/*|/mnt/*|/media/*|/srv/*|/tmp/*) ;; + *) return 1 ;; + esac + # A bare root would make prune's delete set the whole of /mnt. + case "$d" in + /etc/trafficctl/|/mnt/|/media/|/srv/|/tmp/) return 1 ;; + esac + return 0 +} + +hist_dir() { + local d + d=$(uci -q get trafficctl.history.dir 2>/dev/null) + d="${d%/}" + # An absolute path inside an allowed root, or the default. A relative path + # would resolve against whatever directory the caller happened to be in — + # the procd service, rpcd and an interactive shell all differ — and + # silently scatter period files. + hist_validate_dir "$d" || d="$HIST_DIR_DEFAULT" + printf '%s\n' "$d" +} + +hist_period_kind() { + case "$(uci -q get trafficctl.history.period 2>/dev/null)" in + day) printf 'day\n' ;; + week) printf 'week\n' ;; + *) printf 'month\n' ;; + esac +} + +hist_keep() { + local k + k=$(uci -q get trafficctl.history.keep 2>/dev/null) + case "$k" in + ''|*[!0-9]*) k=12 ;; + esac + [ "$k" -lt 1 ] && k=1 + [ "$k" -gt 120 ] && k=120 + printf '%s\n' "$k" +} + +hist_interval() { + local i + i=$(uci -q get trafficctl.history.interval 2>/dev/null) + case "$i" in + ''|*[!0-9]*) i=3600 ;; + esac + # Floor of 300 s. The point of this store is to stay off the flash write + # path; a 10 s interval would reproduce exactly the problem that put the + # live accumulator in tmpfs. + [ "$i" -lt 300 ] && i=300 + printf '%s\n' "$i" +} + +# How often the live accumulator is advanced. Separate from the flush interval +# and much shorter, because of accuracy note 2 above: with a conntrack source, +# anything that starts and ends between two samples is invisible. This costs a +# conntrack read and a tmpfs write, and never touches flash. +hist_sample_interval() { + local i + i=$(uci -q get trafficctl.history.sample_interval 2>/dev/null) + case "$i" in + ''|*[!0-9]*) i=60 ;; + esac + [ "$i" -lt 10 ] && i=10 + [ "$i" -gt 3600 ] && i=3600 + printf '%s\n' "$i" +} + +# Period id for a timestamp. ISO week uses %G with %V, not %Y: in the days +# where the calendar year and the ISO week year disagree, %Y-%V names a week +# that belongs to the other year and two different weeks collide on one id. +hist_period_id() { + local when="${1:-}" kind + kind=$(hist_period_kind) + case "$kind" in + day) set -- '+%Y-%m-%d' ;; + week) set -- '+%G-W%V' ;; + *) set -- '+%Y-%m' ;; + esac + if [ -n "$when" ]; then + date -d "@$when" "$1" 2>/dev/null || date "$1" + else + date "$1" + fi +} + +hist_file() { printf '%s/%s.tsv\n' "$(hist_dir)" "$1"; } + +hist_err_set() { + printf '%s\t%s\n' "$(date +%s)" "$1" > "$ERRFILE" 2>/dev/null + tctl_log config "history: $1" 2>/dev/null || true +} +hist_err_clear() { rm -f "$ERRFILE" 2>/dev/null; } + +# Wrap a durable write so a hung mount cannot park this process for ever. +# BusyBox ships timeout, but not every build enables it, so its absence +# degrades to an unguarded call rather than to no write at all. +hist_guard() { + if command -v timeout >/dev/null 2>&1; then + timeout 15 "$@" + else + "$@" + fi +} + +# "mac:" where a MAC can be established, else "ip:". +# +# IP is the wrong key for history even though it is the right one for live +# monitoring: a device offline for a week comes back on a different lease, and +# its old address gets handed to something else — so IP-keyed history silently +# merges two devices. MAC is stable. +# +# It is not always available. Clients behind a downstream router and anything +# in extra_subnets have no lease and no neighbour entry here, and +# tctl_ip_is_nexthop deliberately refuses a router's own MAC (it is the source +# of every packet it forwards, so trusting it would collapse a whole subnet +# into one row). Those get an ip: key, marked as such, so the UI can say which +# history is stable and which is best-effort. +# +# A device that appears first without a MAC and later with one gets a SECOND +# row rather than having the first adopted. Adoption would be a guess that the +# two are the same device, and the case it would fix (a routed client moving +# onto the local LAN) is rare while the case it would break (an address reused +# by something else) silently merges unrelated traffic. +hist_key() { + local ip="$1" mac + mac=$(tctl_target_mac "$ip" 2>/dev/null) || mac="" + if [ -n "$mac" ]; then + printf 'mac:%s\t%s\n' "$mac" "$mac" + else + printf 'ip:%s\t-\n' "$ip" + fi +} + +# Name precedence: manual alias > DHCP lease > cached reverse DNS — the same +# order trafficctl-summary.sh applies. It is reimplemented here rather than +# shared because summary resolves names for every device in one awk pass on a +# 2 s poll, and this runs once an hour for a handful of changed rows; the +# ORDER is the part that must not diverge. +hist_name() { + local ip="$1" n + n=$(awk -v ip="$ip" -F'[ \t]+' '$1 == ip { $1 = ""; sub(/^[ \t]+/, ""); print; exit }' \ + /etc/trafficctl/names 2>/dev/null) + [ -n "$n" ] && { printf '%s\n' "$n"; return 0; } + n=$(awk -v ip="$ip" '$3 == ip && $4 != "*" { print $4; exit }' \ + "${TCTL_LEASES_FILE:-/tmp/dhcp.leases}" 2>/dev/null) + [ -n "$n" ] && { printf '%s\n' "$n"; return 0; } + n=$(awk -v ip="$ip" -F'\t' '$1 == ip && $2 != "-" { print $2; exit }' \ + /tmp/trafficctl_rdns_cache 2>/dev/null) + [ -n "$n" ] && [ "$n" != "-" ] && { printf '%s\n' "$n"; return 0; } + printf '\n' +} + +# Tabs and newlines would corrupt the row format; a name comes from a DHCP +# lease or a PTR record, neither of which this script controls. +hist_sanitize() { + printf '%s' "$1" | tr -d '\t\r\n' | cut -c1-64 +} + +hist_lock() { + local tries=0 + while ! mkdir "$LOCKD" 2>/dev/null; do + tries=$((tries + 1)) + if [ "$tries" -ge 50 ]; then + # Steal rather than skip for ever: a stale lock from a killed + # flush must not freeze history permanently. Losing one interval + # to an interleaved write is the lesser failure. + rmdir "$LOCKD" 2>/dev/null || true + mkdir "$LOCKD" 2>/dev/null || return 1 + return 0 + fi + sleep 0.1 2>/dev/null || sleep 1 + done + return 0 +} +hist_unlock() { rmdir "$LOCKD" 2>/dev/null || true; } + +# ────────────────────────────────────────────────────────────────────────────── +# flush +# ────────────────────────────────────────────────────────────────────────────── +hist_flush() { + local dir pid file tmp now sample keys ip line rc=0 + + hist_enabled || { printf '{"ok":false,"reason":"disabled"}\n'; return 0; } + + dir=$(hist_dir) + if ! hist_guard mkdir -p "$dir" 2>/dev/null; then + hist_err_set "cannot create $dir" + printf '{"ok":false,"reason":"mkdir failed","dir":"%s"}\n' "$dir" + return 1 + fi + if [ ! -w "$dir" ]; then + hist_err_set "$dir is not writable" + printf '{"ok":false,"reason":"not writable","dir":"%s"}\n' "$dir" + return 1 + fi + + pid=$(hist_period_id) + file=$(hist_file "$pid") + now=$(date +%s) + + hist_lock || { + printf '{"ok":false,"reason":"locked"}\n' + return 1 + } + # shellcheck disable=SC2064 # expand now, not at trap time + trap "hist_unlock" EXIT INT TERM + + sample="/tmp/.trafficctl_history.sample.$$" + tmp="$dir/.history.$pid.$$" + # shellcheck disable=SC2064 + trap "rm -f '$sample' '$tmp'; hist_unlock" EXIT INT TERM + + # --all so a device whose last flow expired still reports the total it has + # accumulated; without it a quiet device's growth would be dropped from + # every flush it happened to be idle for. + /usr/local/bin/trafficctl-totals.sh --all 2>/dev/null \ + | sed 's/},{/}\n{/g' > "$sample" + if [ ! -s "$sample" ]; then + hist_err_set "totals produced nothing" + printf '{"ok":false,"reason":"no sample"}\n' + return 1 + fi + + # Resolve key/mac/name per address in the shell, where tctl_* lives, and + # hand awk a plain table. Doing it the other way round would mean calling + # out of awk per device. + keys="/tmp/.trafficctl_history.keys.$$" + # shellcheck disable=SC2064 + trap "rm -f '$sample' '$tmp' '$keys'; hist_unlock" EXIT INT TERM + : > "$keys" + while IFS= read -r line; do + case "$line" in + *'"ip":"'*) ;; + *) continue ;; + esac + ip=${line#*\"ip\":\"} + ip=${ip%%\"*} + tctl_validate_ip "$ip" || continue + printf '%s\t%s\t%s\n' "$ip" "$(hist_key "$ip")" "$(hist_sanitize "$(hist_name "$ip")")" >> "$keys" + done < "$sample" + + awk -v sample="$sample" -v keys="$keys" -v cursor="$CURSOR" -v out="$tmp" \ + -v store="$file" -v now="$now" ' + function num(line, key, re, seg) { + re = "\"" key "\"[ \t]*:[ \t]*" + if (!match(line, re)) return -1 + seg = substr(line, RSTART + RLENGTH) + if (!match(seg, /^[0-9]+/)) return -1 + return substr(seg, RSTART, RLENGTH) + 0 + } + BEGIN { + FS = "\t" + # Created unconditionally and up front. If this file did not exist on + # a flush that nonetheless rewrote the period file, the mv below would + # fail, the cursor would stay behind, and the NEXT flush would fold the + # same growth in a second time. + cf = cursor ".new" + printf "" > cf + while ((getline l < keys) > 0) { + n = split(l, f, "\t") + if (n < 3) continue + kkey[f[1]] = f[2]; kmac[f[1]] = f[3]; kname[f[1]] = f[4] + } + close(keys) + + # Cursor: key -> what has already been folded in. + while ((getline l < cursor) > 0) { + n = split(l, f, "\t") + if (n < 5) continue + crx[f[1]] = f[2] + 0; ctx[f[1]] = f[3] + 0 + ctc[f[1]] = f[4] + 0; cud[f[1]] = f[5] + 0 + seen_cursor[f[1]] = 1 + } + close(cursor) + + # Existing rows for this period. + while ((getline l < store) > 0) { + if (l ~ /^#/) continue + n = split(l, f, "\t") + if (n < 10) continue + k = f[1] + order[++norder] = k + rmac[k] = f[2]; rip[k] = f[3]; rname[k] = f[4] + rrx[k] = f[5] + 0; rtx[k] = f[6] + 0 + rtc[k] = f[7] + 0; rud[k] = f[8] + 0 + rfirst[k] = f[9] + 0; rlast[k] = f[10] + 0 + rdeg[k] = (n >= 11 ? f[11] : "false") + have[k] = 1 + } + close(store) + + while ((getline line < sample) > 0) { + if (line !~ /"ip":"/) continue + ip = line + sub(/^.*"ip":"/, "", ip) + sub(/".*$/, "", ip) + if (!(ip in kkey)) continue + k = kkey[ip] + + lrx = num(line, "bytes_in_total"); if (lrx < 0) lrx = 0 + ltx = num(line, "bytes_out_total"); if (ltx < 0) ltx = 0 + ltc = num(line, "bytes_tcp_total") + lud = num(line, "bytes_udp_total") + deg = "false" + if (index(line, "\"degraded\":true") > 0) deg = "true" + + # Two addresses can map to ONE key within a flush (a device that + # changed lease while the cursor still remembers both), so the + # live reading for a key is the sum over its addresses, and the + # cursor comparison happens once, after the loop. + arx[k] += lrx; atx[k] += ltx + if (ltc >= 0) { atc[k] += ltc; ktc[k] = 1 } + if (lud >= 0) { aud[k] += lud; kud[k] = 1 } + if (deg == "true") adeg[k] = "true" + amac[k] = kmac[ip]; aip[k] = ip + if (kname[ip] != "") aname[k] = kname[ip] + touched[k] = 1 + } + close(sample) + + for (k in touched) { + lrx = arx[k] + 0; ltx = atx[k] + 0 + ltc = -1 + if (k in ktc) ltc = atc[k] + 0 + lud = -1 + if (k in kud) lud = aud[k] + 0 + + # Growth only. A live value below the cursor means the accumulator + # restarted; rebaseline and add nothing, because the bytes it is + # now below by were already folded in. + drx = lrx + dtx = ltx + if (seen_cursor[k]) { + drx = 0 + dtx = 0 + if (lrx >= crx[k]) drx = lrx - crx[k] + if (ltx >= ctx[k]) dtx = ltx - ctx[k] + } + dtc = -1; dud = -1 + if (ltc >= 0) { + if (!seen_cursor[k] || ctc[k] < 0) dtc = ltc + else dtc = (ltc >= ctc[k]) ? ltc - ctc[k] : 0 + } + if (lud >= 0) { + if (!seen_cursor[k] || cud[k] < 0) dud = lud + else dud = (lud >= cud[k]) ? lud - cud[k] : 0 + } + + if (!(k in have)) { + order[++norder] = k + have[k] = 1 + rfirst[k] = now + rrx[k] = 0; rtx[k] = 0; rtc[k] = 0; rud[k] = 0 + rdeg[k] = "false" + } + rrx[k] += drx; rtx[k] += dtx + if (dtc >= 0) rtc[k] += dtc + if (dud >= 0) rud[k] += dud + if (amac[k] != "" && amac[k] != "-") rmac[k] = amac[k] + rip[k] = aip[k] + # Keep the last name we knew. A device that goes offline loses its + # lease, so re-reading an empty name over a good one would erase + # the only label its history will ever have. + if (aname[k] != "") rname[k] = aname[k] + if (adeg[k] == "true") rdeg[k] = "true" + # Only movement counts as activity. A device sitting idle should + # not keep refreshing last_seen for a whole month. + if (drx > 0 || dtx > 0) rlast[k] = now + else if (rlast[k] == 0) rlast[k] = now + + ncrx[k] = lrx; nctx[k] = ltx; nctc[k] = ltc; ncud[k] = lud + written[k] = 1 + } + + printf "# trafficctl-history %d\n", 1 > out + for (i = 1; i <= norder; i++) { + k = order[i] + if (!(k in have) || emitted[k]) continue + emitted[k] = 1 + nm = rname[k]; if (nm == "") nm = "-" + mc = rmac[k]; if (mc == "") mc = "-" + dg = rdeg[k]; if (dg == "") dg = "false" + printf "%s\t%s\t%s\t%s\t%.0f\t%.0f\t%.0f\t%.0f\t%d\t%d\t%s\n", \ + k, mc, rip[k], nm, rrx[k], rtx[k], rtc[k], rud[k], \ + rfirst[k], rlast[k], dg > out + } + close(out) + + # The cursor keeps keys that were NOT in this sample: a device absent + # for one flush must not have its next reading treated as a fresh + # baseline, or its whole lifetime total would be folded in twice. + for (k in seen_cursor) { + if (k in written) continue + printf "%s\t%.0f\t%.0f\t%.0f\t%.0f\n", k, crx[k], ctx[k], ctc[k], cud[k] > cf + } + for (k in written) { + printf "%s\t%.0f\t%.0f\t%.0f\t%.0f\n", k, ncrx[k], nctx[k], nctc[k], ncud[k] > cf + } + close(cf) + }' /dev/null; then + hist_err_set "cannot write $file" + printf '{"ok":false,"reason":"write failed","file":"%s"}\n' "$file" + return 1 + fi + # The cursor advances ONLY after the period file is safely in place. + # Advancing it first and then failing the write would discard the deltas + # for good; this way the same growth is simply folded in next time. + mv "$CURSOR.new" "$CURSOR" 2>/dev/null || true + printf '%s\n' "$now" > "$STAMP" 2>/dev/null + hist_err_clear + + printf '{"ok":true,"period":"%s","devices":%s,"at":%s}\n' \ + "$pid" "$(grep -vc '^#' "$file" 2>/dev/null || echo 0)" "$now" +} + +# ────────────────────────────────────────────────────────────────────────────── +# read / periods +# ────────────────────────────────────────────────────────────────────────────── +hist_read() { + local dir from to files f + dir=$(hist_dir) + from="$1"; to="$2" + [ -z "$from" ] && from=$(hist_period_id) + [ -z "$to" ] && to="$from" + + files="" + for f in "$dir"/*.tsv; do + [ -f "$f" ] || continue + p=$(basename "$f" .tsv) + # String comparison is the right one here: every period id this script + # writes is fixed-width and zero-padded, so lexical order IS + # chronological order. (That is why the day format is %Y-%m-%d and not + # anything locale-shaped.) + [ "$p" \< "$from" ] && continue + [ "$to" \< "$p" ] && continue + files="$files $f" + done + + # shellcheck disable=SC2086 # deliberate word splitting over the file list + awk -F'\t' ' + FNR == 1 { period = FILENAME; sub(/.*\//, "", period); sub(/\.tsv$/, "", period) } + /^#/ { next } + NF >= 10 { + k = $1 + if (!(k in seen)) { order[++n] = k; seen[k] = 1; first[k] = $9 } + mac[k] = ($2 != "-" ? $2 : mac[k]) + ip[k] = $3 + if ($4 != "-") name[k] = $4 + rx[k] += $5; tx[k] += $6; tc[k] += $7; ud[k] += $8 + if ($9 + 0 < first[k] + 0) first[k] = $9 + if ($10 + 0 > last[k] + 0) last[k] = $10 + if ($11 == "true") deg[k] = 1 + periods[k] = periods[k] (index(periods[k], "\"" period "\"") ? "" : (periods[k] == "" ? "" : ",") "\"" period "\"") + } + END { + printf "[" + for (i = 1; i <= n; i++) { + k = order[i] + nm = "" + if (k in name) nm = name[k] + kind = "ip" + if (k ~ /^mac:/) kind = "mac" + if (i > 1) printf "," + printf "{\"key\":\"%s\",\"key_kind\":\"%s\",\"mac\":\"%s\",\"ip\":\"%s\",\"name\":\"%s\",\"bytes_in\":%.0f,\"bytes_out\":%.0f,\"bytes_tcp\":%.0f,\"bytes_udp\":%.0f,\"first_seen\":%d,\"last_seen\":%d,\"degraded\":%s,\"periods\":[%s]}", \ + k, kind, (mac[k] == "" ? "-" : mac[k]), ip[k], nm, \ + rx[k], tx[k], tc[k], ud[k], first[k], last[k], \ + (k in deg ? "true" : "false"), periods[k] + } + printf "]\n" + }' $files /dev/null || printf '[]\n' +} + +hist_periods() { + local dir f p + dir=$(hist_dir) + printf '[' + n=0 + for f in "$dir"/*.tsv; do + [ -f "$f" ] || continue + p=$(basename "$f" .tsv) + [ "$n" -gt 0 ] && printf ',' + printf '{"period":"%s","devices":%s,"bytes":%s}' \ + "$p" \ + "$(grep -vc '^#' "$f" 2>/dev/null || echo 0)" \ + "$(awk -F'\t' '!/^#/ && NF >= 10 { s += $5 + $6 } END { printf "%.0f", s + 0 }' "$f" 2>/dev/null || echo 0)" + n=$((n + 1)) + done + printf ']\n' +} + +hist_prune() { + local dir keep pid total f dropped=0 + dir=$(hist_dir) + keep=$(hist_keep) + pid=$(hist_period_id) + # The current period is never a retention candidate — it is still being + # written to, and "keep 1" must not mean "delete the data you are + # collecting right now". + # shellcheck disable=SC2046 # deliberate splitting: one word per period id + set -- $(find "$dir" -name '*.tsv' -type f 2>/dev/null \ + | sed 's|.*/||; s|\.tsv$||' | grep -vxF "$pid" | sort) + total=$# + while [ "$total" -gt "$keep" ] && [ "$#" -gt 0 ]; do + f="$dir/$1.tsv" + rm -f "$f" 2>/dev/null && dropped=$((dropped + 1)) + total=$((total - 1)) + shift + done + printf '{"ok":true,"kept":%s,"dropped":%s}\n' "$keep" "$dropped" +} + +# What the numbers in the store are worth right now, as one word, derived from +# the live offload mode. Reported so a consumer can label the view instead of +# presenting a lower bound as a measurement. +# +# degraded — uncountered offload, no nft fallback: a LOWER BOUND +# counter — nft counter maps: exact, no protocol split +# sampled — conntrack: exact to within flows shorter than sample_interval +hist_accuracy() { + local mode src + mode=$(tctl_get_offload_mode 2>/dev/null) + # What the sampler actually settled on last time it ran, which is the + # authority on whether the nft fallback was available — the offload mode + # alone cannot tell us that. + src=$(awk '{ print $11; exit }' "$TOTALS_STATE" 2>/dev/null) + case "$mode" in + hardware|software) + if [ "$src" = "nft" ]; then printf 'counter\n'; else printf 'degraded\n'; fi + ;; + *) + if [ "$src" = "nft" ]; then printf 'counter\n'; else printf 'sampled\n'; fi + ;; + esac +} + +hist_status() { + local dir en pid last err errat w + dir=$(hist_dir) + en=false + hist_enabled && en=true + pid=$(hist_period_id) + last=$(cat "$STAMP" 2>/dev/null) + case "$last" in ''|*[!0-9]*) last=0 ;; esac + err=""; errat=0 + if [ -f "$ERRFILE" ]; then + errat=$(cut -f1 "$ERRFILE" 2>/dev/null) + err=$(cut -f2- "$ERRFILE" 2>/dev/null) + case "$errat" in ''|*[!0-9]*) errat=0 ;; esac + fi + w=false + [ -d "$dir" ] && [ -w "$dir" ] && w=true + printf '{"enabled":%s,"dir":"%s","writable":%s,"period":"%s","period_kind":"%s","interval":%s,"sample_interval":%s,"keep":%s,"format":%s,"last_flush":%s,"accuracy":"%s","offload":"%s","error":"%s","error_at":%s}\n' \ + "$en" "$dir" "$w" "$pid" "$(hist_period_kind)" \ + "$(hist_interval)" "$(hist_sample_interval)" "$(hist_keep)" "$FORMAT" "$last" \ + "$(hist_accuracy)" "$(tctl_get_offload_mode 2>/dev/null)" \ + "$(printf '%s' "$err" | tr -d '\042\134')" "$errat" +} + +# The procd service loop. Two cadences in one process, which is the whole +# reason this is a service and not a cron line: sampling has to be frequent +# (accuracy note 2 — a conntrack flow shorter than the sample interval is +# invisible) and costs only a tmpfs write, while flushing is rare because it +# touches durable storage. +# +# Re-reads its own configuration every tick rather than caching it at start: +# changing the interval in the UI should take effect without the operator +# having to know there is a service to restart. +hist_keeper() { + local last=0 now interval + + # Pick up where a previous run left off, so a restart does not reset the + # flush clock and silently stretch the gap to the durable store. + last=$(cat "$STAMP" 2>/dev/null) + case "$last" in ''|*[!0-9]*) last=0 ;; esac + + while :; do + if hist_enabled; then + /usr/local/bin/trafficctl-totals.sh --all >/dev/null 2>&1 + now=$(date +%s) + interval=$(hist_interval) + # A clock that jumped backwards (NTP settling after boot, which is + # the norm on a router with no RTC) would otherwise park the next + # flush an interval into the past's future and stall the store. + [ "$now" -lt "$last" ] && last=0 + if [ $((now - last)) -ge "$interval" ]; then + hist_flush >/dev/null 2>&1 + hist_prune >/dev/null 2>&1 + last=$now + fi + fi + sleep "$(hist_sample_interval)" + done +} + +case "$1" in + sample) + # Advance the accumulator and throw the JSON away. Cheap, tmpfs only, + # and the thing that makes an hourly flush mean anything. + hist_enabled || { printf '{"ok":false,"reason":"disabled"}\n'; exit 0; } + /usr/local/bin/trafficctl-totals.sh --all >/dev/null 2>&1 + printf '{"ok":true,"at":%s}\n' "$(date +%s)" + ;; + flush) hist_flush ;; + keeper) hist_keeper ;; + # Exposed so the rpcd setter validates against THIS implementation rather + # than a second copy of the rule that can drift away from it. + validate-dir) hist_validate_dir "${2%/}" ;; + read) hist_read "$2" "$3" ;; + periods) hist_periods ;; + prune) hist_prune ;; + status) hist_status ;; + *) + printf 'Usage: %s {sample|flush|keeper|validate-dir |read [period [to]]|periods|prune|status}\n' "$0" >&2 + exit 1 + ;; +esac diff --git a/luci-app-trafficctl/root/usr/share/luci/menu.d/luci-app-trafficctl.json b/luci-app-trafficctl/root/usr/share/luci/menu.d/luci-app-trafficctl.json index 5d7613106..4541b732a 100644 --- a/luci-app-trafficctl/root/usr/share/luci/menu.d/luci-app-trafficctl.json +++ b/luci-app-trafficctl/root/usr/share/luci/menu.d/luci-app-trafficctl.json @@ -23,9 +23,21 @@ } }, + "admin/status/trafficctl/history": { + "title": "History", + "order": 2, + "action": { + "type": "view", + "path": "trafficctl/history" + }, + "depends": { + "acl": [ "luci-app-trafficctl" ] + } + }, + "admin/status/trafficctl/portfw": { "title": "Port Forwards", - "order": 2, + "order": 3, "action": { "type": "view", "path": "trafficctl/portfw" diff --git a/luci-app-trafficctl/root/usr/share/rpcd/acl.d/luci-app-trafficctl.json b/luci-app-trafficctl/root/usr/share/rpcd/acl.d/luci-app-trafficctl.json index 018d23d39..c4c5849b5 100644 --- a/luci-app-trafficctl/root/usr/share/rpcd/acl.d/luci-app-trafficctl.json +++ b/luci-app-trafficctl/root/usr/share/rpcd/acl.d/luci-app-trafficctl.json @@ -10,7 +10,9 @@ "portfw_list", "names_list", "netify_status", "netify_list", "shape_status", "config_get", "telegram_config_get", - "logging_config_get", "newdevice_config_get" + "logging_config_get", "newdevice_config_get", + "history_status", "history_read", "history_periods", + "history_config_get" ], "file": [ "read" ] }, @@ -30,7 +32,8 @@ "telegram_config_set", "telegram_test", "logging_config_set", "activity_log", "newdevice_config_set", - "cut_status", "cut_set" + "cut_status", "cut_set", + "history_config_set", "history_flush" ] }, "uci": [ "trafficctl" ]