From 52c8ed9e4a559333faf91df5fc88fb13dbf1a2f3 Mon Sep 17 00:00:00 2001 From: TheBigHNF Date: Fri, 27 Feb 2015 01:36:24 -0600 Subject: [PATCH 1/3] Added LUA version --- README.md | 74 ++++++++----- init.lua | 273 ++++++++++++++++++++++++++++++++++++++++++++++ metrics.json | 132 ++++++++++++++++++++++ modules/tools.lua | 32 ++++++ plugin.json | 2 + 5 files changed, 485 insertions(+), 28 deletions(-) create mode 100644 init.lua create mode 100644 metrics.json create mode 100644 modules/tools.lua diff --git a/README.md b/README.md index fd9e3b8..c7096b5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Boundary Litespeed Plugin ------------------------- -Collects metrics from Litespeed HTTP Server instance. +Collects metrics from LiteSpeed HTTP Web-Server. ### Prerequisites @@ -9,44 +9,62 @@ Collects metrics from Litespeed HTTP Server instance. |:----------|:-----:|:-------:|:-------:|:----:| | Supported | v | - | v | v | +- **OS**: Tested to work on **Debian-based Linux distributions** (although any Linux-based OS should work). +- Requires access to "/tmp/lshttpd" (LiteSpeed real-time reporting folder). -| Runtime | node.js | Python | Java | +#### **Meter V4.0 or greater** + +To get the new meter: + + curl -fsS \ + -d "{\"token\":\"\"}" \ + -H "Content-Type: application/json" \ + "https://meter.boundary.com/setup_meter" > setup_meter.sh + chmod +x setup_meter.sh + ./setup_meter.sh + +| Runtime | node.js | Python | Java | +|:---------|:-------:|:------:|:----:| +| Required | | | | + +#### Meter less than V4.0 + +| Runtime | node.js | Python | Java | |:---------|:-------:|:------:|:----:| | Required | + | | | - [How to install node.js?](https://help.boundary.com/hc/articles/202360701) ### Plugin Setup -None +None required. #### Plugin Configuration Fields - Litespeed writes server statistics to multiple report files in the /tmp/lshttpd folder by default, check your configuration to ensure that the path is correct. The number of reports are based on the number of CPUs on your server and the number of CPUs that you have licences for. -|Field Name |Description | -|:------------|:---------------------------------------| -|Report path |The path to the lshttp .rtreport files | -|Virtual Hosts|Include individual VHosts in your graphs| +|Field Name |Identifier |Type |Description | +|:--------------|:------------|--------------|:---------------------------------------------------------------------------------| +|Report Path |reportPath |string |The path to the LiteSpeed '.rtreport' file(s) to watch (default: '/tmp/lshttpd'). | +|Virtual Hosts |virtualHosts |array[string] |Include individual VHosts in your graphs. | +|Poll Interval |pollInterval |integer |How often (in milliseconds) to poll the metrics file(s) (default: 5000). | ### Metrics Collected +Tracks the following metrics for the [LiteSpeed](http://www.litespeedtech.com/) HTTP Web-Server. -Tracks the following metrics for the [litespeed](http://www.litespeedtech.com/) webserver. - -|Metric Name |Description | -|:-------------------------------|:-------------------------------------------| -|Litespeed HTTP Connection limit |The ratio of connections to the Max Limit | -|Litespeed Http Conns |The number of current HTTP connections | -|Litespeed Idle Http Conns |The number of current idle HTTP connections | -|Litespeed HTTPS Connection limit|The ratio of connections to the Max Limit | -|Litespeed Https Conns |The number of current HTTPS connections | -|Litespeed Idle Https Conns |The number of current idle HTTPS connections| -|Litespeed Http Bytes In |The amount of inbound data over HTTP | -|Litespeed Http Bytes Out |The amount of outbound data over HTTP | -|Litespeed Https Bytes In |The amount of inbound data over HTTPS | -|Litespeed Https Bytes Out |The amount of outbound data over HTTPS | -|Litespeed Total Bytes In |The total amount of inbound data | -|Litespeed Total Bytes Out |The total amount of outbound data | -|Litespeed Cache Hits |The number of cache hits | -|Litespeed Cache Hit Ratio |The ratio of cache hits to overall requests | -|Litespeed Requests in Process |The number of requests in process | -|Litespeed Requests |The number of requests | +|Metric Name |Description | +|:--------------------------------|:---------------------------------------------| +|Litespeed HTTP Connection limit |The ratio of connections to the Max Limit. | +|Litespeed HTTP Connections |The number of current HTTP connections. | +|Litespeed Idle HTTP Connections |The number of current idle HTTP connections. | +|Litespeed HTTPS Connection limit |The ratio of connections to the Max Limit. | +|Litespeed HTTPS Connections |The number of current HTTPS connections. | +|Litespeed Idle HTTPS Connections |The number of current idle HTTPS connections. | +|Litespeed HTTP Bytes In |The amount of inbound data over HTTP. | +|Litespeed HTTP Bytes Out |The amount of outbound data over HTTP. | +|Litespeed HTTPS Bytes In |The amount of inbound data over HTTPS. | +|Litespeed HTTPS Bytes Out |The amount of outbound data over HTTPS. | +|Litespeed Total Bytes In |The total amount of inbound data. | +|Litespeed Total Bytes Out |The total amount of outbound data. | +|Litespeed Cache Hits |The number of cache hits. | +|Litespeed Cache Hit Ratio |The ratio of cache hits to overall requests. | +|Litespeed Requests in Process |The number of requests in process. | +|Litespeed Requests |The number of requests. | diff --git a/init.lua b/init.lua new file mode 100644 index 0000000..f984616 --- /dev/null +++ b/init.lua @@ -0,0 +1,273 @@ +-- +-- [boundary.com] Couchbase Lua Plugin +-- [author] Valeriu Paloş +-- + +-- +-- Imports. +-- +local fs = require('fs') +local json = require('json') +local http = require('http') +local os = require('os') +local path = require('path') +local timer = require('timer') +local tools = require('tools') +local url = require('url') + +-- +-- Initialize. +-- +local _buckets = {} +local _parameters = json.parse(fs.readFileSync('param.json')) or {} + +local _reportPath = _parameters.reportPath or '/tmp/lshttpd' +local _virtualHosts = _parameters.virtualHosts or {} +local _pollInterval = tools.fence(tonumber(_parameters.pollInterval) or 5000, 100, 1000 * 60 * 60 * 24) + +local _ignoredLines = { '^VERSION', '^UPTIME', '^BLOCKED_IP', '^EOF' } + +-- +-- Metrics source. +-- +local _source = + (type(_parameters.source) == 'string' and _parameters.source:gsub('%s+', '') ~= '' and _parameters.source) or + os.hostname() + +-- +-- Scan paths. +-- +local reports = {} +local file = path.join(_reportPath, '.rtreport') +if fs.existsSync(file) then + table.insert(reports, file) +end +for i = 0, #os.cpus() do + local file = path.join(_reportPath, '.rtreport' .. i) + if fs.existsSync(file) then + table.insert(reports, file) + end +end + +-- +-- Scan virtual hosts. +-- +local virtualHosts = {} +if #_parameters.virtualHosts == 0 then + table.insert(_parameters.virtualHosts, '') +end +for _, item in ipairs(_parameters.virtualHosts) do + if item then + local vhost = item:match('^([^|]*)') + local alias = item:match('|(.*)$') + + if virtualHosts[vhost] then + error(("The virtual host '%s' is defined twice and hosts must be distinct!"):format(vhost)) + end + + virtualHosts[vhost] = _source .. '-' .. tools.trim(alias or vhost) + end +end + +-- +-- Calculus utility functions. +-- +function sum(a, b) + return (a and b and math.max(a + b, 0)) or a or b +end +function delta(a, b) + local na, nb = tonumber(a), tonumber(b) + return (na and nb and math.max(na - nb, 0)) or 0 +end +function parseFloat(x) + return tonumber(x) or 0 +end +function parseKeyValues(data) + local result = {} + for key, value in data:gmatch("([%w_]+):%s*(%d+)") do + result[key:upper()] = parseFloat(value) + end + return result +end + +-- +-- Schedule poll. +-- +function schedule() + timer.setTimeout(_pollInterval, poll) +end + +-- +-- Print a metric. +-- +function metric(stamp, id, value, source) + print(string.format('%s %s %s %d', id, value, source or _source, stamp)) +end + +-- +-- Parse a report file. +-- +function parseReport(file, callback) + + -- Sample Report: + -- + -- VERSION: LiteSpeed Web Server/Standard/4.2.21 + -- UPTIME: 02:38:00 + -- BPS_IN: 0, BPS_OUT: 0, SSL_BPS_IN: 0, SSL_BPS_OUT: 0 + -- MAXCONN: 150, MAXSSL_CONN: 150, PLAINCONN: 0, AVAILCONN: 150, IDLECONN: 0, SSLCONN: 0, AVAILSSL: 150 + -- REQ_RATE []: REQ_PROCESSING: 0, REQ_PER_SEC: 0.0, TOT_REQS: 0, CACHE_HITS_PER_SEC: 0.0, TOTAL_CACHE_HITS: 0 + -- REQ_RATE [Example]: REQ_PROCESSING: 0, REQ_PER_SEC: 0.0, TOT_REQS: 0, CACHE_HITS_PER_SEC: 0.0, TOTAL_CACHE_HITS: 0 + -- REQ_RATE [_AdminVHost]: REQ_PROCESSING: 0, REQ_PER_SEC: 0.0, TOT_REQS: 0, CACHE_HITS_PER_SEC: 0.0, TOTAL_CACHE_HITS: 0 + -- BLOCKED_IP: + -- EOF + -- + fs.readFile(file, function(failure, data) + + -- Safety check. + if failure or not data then + print("ERROR: " .. failure) + return callback({}) + end + + -- Walk through lines. + local results = { ['virtualHosts'] = {} } + for line in tools.lines(data) do + + -- Catch ignored lines. + local valid = true + for _, ignored in ipairs(_ignoredLines) do + if line:find(ignored) then + valid = false + break + end + end + + -- Parse valid lines. + if valid then + local sink = results + local data = line + + -- General metrics or virtual-host specifc? + local vhost, metrics = line:match('^REQ_RATE %[(.-)%]:(.*)') + if vhost then + data = '' + + if virtualHosts[vhost] then + results.virtualHosts[vhost] = results.virtualHosts[vhost] or {} + sink = results.virtualHosts[vhost] + data = metrics + end + end + + -- Parse metrics. + for key, value in pairs(parseKeyValues(data)) do + sink[key] = (sink[key] or 0) + value + end + end + + end + + callback(results) + end) +end + +-- +-- Collect and aggregate reports. +-- +function collectReports(callback) + local sets = {} + local remaining = #reports + + -- Add results to the stack. + function aggregate(values) + table.insert(sets, values) + if remaining == 0 then + + -- Aggregate if all reports are parsed. + local store = { ['virtualHosts'] = {} } + for _, metrics in ipairs(sets) do + + -- Walk generic values. + for name, value in pairs(metrics) do + if name ~= 'virtualHosts' then + store[name] = (store[name] or 0) + value + end + end + + -- Walk virtual-host-specific values. + for vhost, vhMetrics in pairs(metrics.virtualHosts) do + store.virtualHosts[vhost] = store.virtualHosts[vhost] or {} + for name, value in pairs(vhMetrics) do + store.virtualHosts[vhost][name] = (store.virtualHosts[vhost][name] or 0) + value + end + end + + end + + callback(store) + end + end + + -- Trigger asynchronous parsing of report files. + for _, report in ipairs(reports) do + parseReport(report, function(values) + remaining = remaining - 1 + aggregate(values) + end) + end + +end + +-- +-- Produce metrics. +-- +function poll() + local stamp = os.time() + + -- Trigger one collection. + collectReports(function(metrics) + local httpConnLimit = (metrics.MAXCONN == 0 and 0) or (metrics.PLAINCONN / metrics.MAXCONN) + local httpsConnLimit = (metrics.MAXSSL_CONN == 0 and 0) or (metrics.SSLCONN / metrics.MAXSSL_CONN) + + metrics.BPS_IN = metrics.BPS_IN * 1024; + metrics.BPS_OUT = metrics.BPS_OUT * 1024; + metrics.SSL_BPS_IN = metrics.SSL_BPS_IN * 1024; + metrics.SSL_BPS_OUT = metrics.SSL_BPS_OUT * 1024; + + -- Generic metrics. + metric(stamp, 'LITESPEED_HTTP_CONNECTION_LIMIT', httpConnLimit) + metric(stamp, 'LITESPEED_HTTP_CONNECTIONS', metrics.PLAINCONN) + metric(stamp, 'LITESPEED_HTTP_IDLE_CONNECTIONS', metrics.IDLECONN) + metric(stamp, 'LITESPEED_HTTPS_CONNECTION_LIMIT', httpsConnLimit) + metric(stamp, 'LITESPEED_HTTPS_CONNECTIONS', metrics.SSLCONN) + metric(stamp, 'LITESPEED_HTTPS_IDLE_CONNECTIONS', delta(metrics.AVAILSSL, metrics.SSLCONN)) + metric(stamp, 'LITESPEED_HTTP_BYTES_IN', metrics.BPS_IN) + metric(stamp, 'LITESPEED_HTTP_BYTES_OUT', metrics.BPS_OUT) + metric(stamp, 'LITESPEED_HTTPS_BYTES_IN', metrics.SSL_BPS_IN) + metric(stamp, 'LITESPEED_HTTPS_BYTES_OUT', metrics.SSL_BPS_OUT) + metric(stamp, 'LITESPEED_TOTAL_BYTES_IN', sum(metrics.BPS_IN, metrics.SSL_BPS_IN)) + metric(stamp, 'LITESPEED_TOTAL_BYTES_OUT', sum(metrics.BPS_OUT, metrics.SSL_BPS_OUT)) + + -- Virtual-host specific metrics. + for vhost, vhMetrics in pairs(metrics.virtualHosts) do + local virtualHost = metrics.virtualHosts[vhost]; + local virtualName = virtualHosts[vhost]; + + local cacheHits = virtualHost.CACHE_HITS_PER_SEC or 0; + local requests = virtualHost.REQ_PER_SEC; + local requestsInProcess = virtualHost.REQ_PROCESSING or 0; + local cacheRatio = (requests == 0 and 0) or (cacheHits / requests); + + metric(stamp, 'LITESPEED_CACHE_HITS', cacheHits, virtualName) + metric(stamp, 'LITESPEED_CACHE_RATIO', cacheRatio, virtualName) + metric(stamp, 'LITESPEED_REQUESTS', requests, virtualName) + metric(stamp, 'LITESPEED_REQUESTS_IN_PROCESS', requestsInProcess, virtualName) + end + + -- Reschedule. + schedule() + end) +end + +-- Trigger polling. +poll() \ No newline at end of file diff --git a/metrics.json b/metrics.json new file mode 100644 index 0000000..b0be588 --- /dev/null +++ b/metrics.json @@ -0,0 +1,132 @@ +{ + "result": [ + { + "name": "LITESPEED_HTTP_CONNECTION_LIMIT", + "displayName": "Litespeed HTTP Connection limit", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "number", + "displayNameShort": "The ratio of connections to the Max Limit.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_HTTP_CONNECTIONS", + "displayName": "Litespeed HTTP Connections", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "integer", + "displayNameShort": "The number of current HTTP connections.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_HTTP_IDLE_CONNECTIONS", + "displayName": "Litespeed Idle HTTP Connections", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "integer", + "displayNameShort": "The number of current idle HTTP connections.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_HTTPS_CONNECTION_LIMIT", + "displayName": "Litespeed HTTPS Connection limit", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "number", + "displayNameShort": "The ratio of connections to the Max Limit.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_HTTPS_CONNECTIONS", + "displayName": "Litespeed HTTPS Connections", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "integer", + "displayNameShort": "The number of current HTTPS connections.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_HTTPS_IDLE_CONNECTIONS", + "displayName": "Litespeed Idle HTTPS Connections", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "integer", + "displayNameShort": "The number of current idle HTTPS connections.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_HTTP_BYTES_IN", + "displayName": "Litespeed HTTP Bytes In", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "bytecount", + "displayNameShort": "The amount of inbound data over HTTP.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_HTTP_BYTES_OUT", + "displayName": "Litespeed HTTP Bytes Out", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "bytecount", + "displayNameShort": "The amount of outbound data over HTTP.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_HTTPS_BYTES_IN", + "displayName": "Litespeed HTTPS Bytes In", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "bytecount", + "displayNameShort": "The amount of inbound data over HTTPS.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_HTTPS_BYTES_OUT", + "displayName": "Litespeed HTTPS Bytes Out", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "bytecount", + "displayNameShort": "The amount of outbound data over HTTPS.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_TOTAL_BYTES_IN", + "displayName": "Litespeed Total Bytes In", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "bytecount", + "displayNameShort": "The total amount of inbound data.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_TOTAL_BYTES_OUT", + "displayName": "Litespeed Total Bytes Out", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "bytecount", + "displayNameShort": "The total amount of outbound data.", + "defaultAggregate": "avg" + }, + { + "name": "LITESPEED_CACHE_HITS", + "displayName": "Litespeed Cache Hits", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "integer", + "displayNameShort": "The number of cache hits.", + "defaultAggregate": "avg" + } + { + "name": "LITESPEED_CACHE_RATIO", + "displayName": "Litespeed Cache Hit Ratio", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "number", + "displayNameShort": "The ratio of cache hits to overall requests.", + "defaultAggregate": "avg" + } + { + "name": "LITESPEED_REQUESTS", + "displayName": "Litespeed Requests", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "integer", + "displayNameShort": "The number of requests.", + "defaultAggregate": "avg" + } + { + "name": "LITESPEED_REQUESTS_IN_PROCESS", + "displayName": "Litespeed Requests in Process", + "description": "Total quota of memory allocated by the Couchbase cluster for this node.", + "unit": "integer", + "displayNameShort": "The number of requests in process.", + "defaultAggregate": "avg" + } + ] +} \ No newline at end of file diff --git a/modules/tools.lua b/modules/tools.lua new file mode 100644 index 0000000..7e870bc --- /dev/null +++ b/modules/tools.lua @@ -0,0 +1,32 @@ +-- +-- Tools Module. +-- [author] Valeriu Paloş +-- +local tools = {} + +-- +-- Trim string from both ends. +-- +tools.trim = function(text) + return (text:gsub("^%s*(.-)%s*$", "%1")) +end + +-- +-- Iterate through string line by line. +-- +tools.lines = function(text) + return text:gmatch("[^\n\r]+") +end + +-- +-- Limit a given number x between two boundaries. +-- Either min or max can be nil, to fence on one side only. +-- +tools.fence = function(x, min, max) + return (min and x < min and min) or (max and x > max and max) or x +end + +-- +-- Export. +-- +return tools \ No newline at end of file diff --git a/plugin.json b/plugin.json index 0023c8a..15813f8 100644 --- a/plugin.json +++ b/plugin.json @@ -2,7 +2,9 @@ "description" : "Displays important litespeed metrics", "icon" : "icon.png", "command" : "node index.js", + "command_lua": "boundary-meter init.lua", "postExtract" : "npm install", + "postExtract_lua": "", "ignore" : "node_modules", "metrics" : [ From da217bcc55878586f1c23b0927cc4d3b55ba22e5 Mon Sep 17 00:00:00 2001 From: TheBigHNF Date: Mon, 16 Mar 2015 14:59:02 -0500 Subject: [PATCH 2/3] Fixed error handling to go to stderr --- init.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init.lua b/init.lua index f984616..d7ab847 100644 --- a/init.lua +++ b/init.lua @@ -125,7 +125,7 @@ function parseReport(file, callback) -- Safety check. if failure or not data then - print("ERROR: " .. failure) + process.stderr:write("ERROR: " .. failure) return callback({}) end @@ -270,4 +270,4 @@ function poll() end -- Trigger polling. -poll() \ No newline at end of file +poll() From 0f8377cc9bc84d64ff8710096e2cf8356aeb3c8e Mon Sep 17 00:00:00 2001 From: TheBigHNF Date: Sun, 10 May 2015 22:07:43 -0500 Subject: [PATCH 3/3] Updated README --- README.md | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c7096b5..da20671 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -Boundary Litespeed Plugin -------------------------- +# Boundary Litespeed Plugin + Collects metrics from LiteSpeed HTTP Web-Server. @@ -12,16 +12,10 @@ Collects metrics from LiteSpeed HTTP Web-Server. - **OS**: Tested to work on **Debian-based Linux distributions** (although any Linux-based OS should work). - Requires access to "/tmp/lshttpd" (LiteSpeed real-time reporting folder). -#### **Meter V4.0 or greater** - -To get the new meter: +#### Boundary Meter Versions V4.0 Or Later - curl -fsS \ - -d "{\"token\":\"\"}" \ - -H "Content-Type: application/json" \ - "https://meter.boundary.com/setup_meter" > setup_meter.sh - chmod +x setup_meter.sh - ./setup_meter.sh +- To install new meter go to Settings->Installation or [see instructons|https://help.boundary.com/hc/en-us/sections/200634331-Installation]. +- To upgrade the meter to the latest version - [see instructons|https://help.boundary.com/hc/en-us/articles/201573102-Upgrading-the-Boundary-Meter]. | Runtime | node.js | Python | Java | |:---------|:-------:|:------:|:----:|