Commit d6f4d530 authored by xa's avatar xa

documentation

parent 73dbdaaf
......@@ -40,6 +40,11 @@ $ xba fake
## Tests
```bash
$ pip install -r requirements-tests.txt
$ py.test tests
```
- tests/test_alerting.py explain how to send alerts / recovering events
- tests/test_checks.py unit test for checks
- tests/test_core.py unit test that show how new events are dispatched, and previous events are kept
......
import pytest
from itertools import permutations
from xba.util import merge, switch, JSONRef
from xba.util import merge, swap, JSONRef
@pytest.mark.parametrize("a, b, reverse, expected", [
......@@ -11,8 +11,8 @@ from xba.util import merge, switch, JSONRef
(None, 2, True, (2, None)),
(None, 2, False, (None, 2)),
])
def test_switch(a, b, reverse, expected):
assert switch(a, b, reverse) == expected
def test_swap(a, b, reverse, expected):
assert swap(a, b, reverse) == expected
@pytest.mark.parametrize("a, b, expected", [
......
......@@ -7,23 +7,34 @@ State = namedtuple('State', 'value triggered resolved')
class CheckDispatcher:
"""Check dispatcher.
"""
def __init__(self):
self.checks = []
def register(self, extract, check):
def register(self, extract, emit):
"""
Parameters:
extract (callable): extract value from metrics
emit (callable): emit events based on extracted value
"""
if isinstance(extract, str):
extract = JSONRef(extract)
self.checks.append((extract, check))
self.checks.append((extract, emit))
def dispatch(self, metrics):
date = datetime.now()
events = []
for extract, check in self.checks:
value = extract(metrics)
result = check(value, date)
if isinstance(result, Event):
events.append(result)
for extract, emit in self.checks:
try:
value = extract(metrics)
result = emit(value, date)
except (KeyError, IndexError) as error:
events.append(Event("missing", "missing value", self))
else:
if isinstance(result, Event):
events.append(result)
return events
......@@ -53,22 +64,24 @@ class HitsCheck:
elif self.state.triggered + self.sustainability < date:
return self.alert(value, date)
elif self.state and self.state.triggered + self.sustainability <= date:
# we resolved alert!
return self.ok(value, date)
# we recover from alert
return self.recover(value, date)
def alert(self, value, date):
self.state = State(value, date, None)
return Event(
"alert",
"%s generated an alert - hits = %s, triggered at %s" % (self.name, value, date.strftime("%H:%M:%S")),
"%s generated an alert - hits = %s, triggered at %s" % (
self.name, value, date.strftime("%H:%M:%S")),
self
)
def ok(self, value, date):
def recover(self, value, date):
self.state = None
return Event(
"recovering",
"%s recovering alert - hits = %s, triggered at %s" % (self.name, value, date.strftime("%H:%M:%S")),
"%s recovering alert - hits = %s, triggered at %s" % (
self.name, value, date.strftime("%H:%M:%S")),
self
)
......
......@@ -8,8 +8,12 @@ class State:
"""
def __init__(self, events=200):
"""
Parameters:
events (int): cap to last n events
"""
self.metrics = {}
self._events = deque([], maxlen=events) # cap to last n events
self._events = deque([], maxlen=events)
self.since = None
self.until = None
self.registry = set()
......
......@@ -4,6 +4,8 @@ from tempfile import NamedTemporaryFile
class FakeLogs:
"""Utility that generate a fake logfile
"""
clients = ['0.0.0.0', '1.0.0.0', '2.0.0.0', '3.0.0.0', '4.0.0.0', '5.0.0.0']
resources = ['/', '/foo', '/bar', '/baz', '/qux']
......@@ -12,13 +14,13 @@ class FakeLogs:
def __init__(self, *, loop=None):
self.loop = loop or asyncio.get_event_loop()
self.log = NamedTemporaryFile(mode='w+')
self.file = NamedTemporaryFile(mode='w+')
self.running = False
self.task = None
@property
def filename(self):
return self.log.name
return self.file.name
def start(self):
self.running = True
......@@ -42,6 +44,6 @@ class FakeLogs:
random.choice(self.statuses),
random.randint(0, maxbytes)
)
self.log.write(log)
self.log.flush()
self.file.write(log)
self.file.flush()
await asyncio.sleep(random.random())
......@@ -12,6 +12,8 @@ def sizeof_fmt(num, suffix='B'):
class ConsoleFormatter:
"""Console formatter.
"""
def __init__(self):
curses.initscr()
......
......@@ -5,16 +5,7 @@ CommonLog = namedtuple('CommonLog', 'host ident authuser date request status byt
Request = namedtuple('Request', 'method path query protocol')
class Parser:
def parse(self, log):
return log
def __call__(self, *args, **kwargs):
return self.parse(*args, **kwargs)
class CommonLogFormatParser(Parser):
class CommonLogFormatParser:
def parse(self, log):
host, ident, authuser, remains = log.split(None, 3)
......@@ -34,6 +25,8 @@ class CommonLogFormatParser(Parser):
except Exception as error:
raise ParseException("cannot parse log %s: bad format", log) from error
__call__ = parse
def parse_host(self, data):
return None if data == "-" else data
......
......@@ -2,6 +2,7 @@ from collections import defaultdict
from collections import deque
from datetime import datetime
from xba import util
from xba.logs import CommonLog
class Metrics:
......@@ -22,7 +23,6 @@ class Metrics:
total_clients (int): total clients (read only)
total_hits (int): total hits
total_sections (int): total sections (read only)
since (datetime): date of first increment
"""
......@@ -43,7 +43,6 @@ class Metrics:
def increment(self, section, client, method, bytes, status_range):
self.since = self.since or datetime.now()
self.bytes_per_section[section] += bytes
self.bytes_per_client[client] += bytes
self.clients.add(client)
......@@ -65,8 +64,12 @@ class Metrics:
def total_sections(self):
return len(self.sections)
@property
def total(self):
return self.extract()
def extract(self):
"""Extract all data
"""Extract and format values.
"""
sections = {}
for section in self.sections:
......@@ -118,7 +121,7 @@ class MetricsAggregator:
if self.rt.since is None:
return
since, now = self.rt.since, datetime.now()
metrics, self.rt = self.rt.extract(), Metrics(since=now)
metrics, self.rt = self.rt.total, Metrics(since=now)
self.history.append((since, now, metrics))
@property
......@@ -126,7 +129,7 @@ class MetricsAggregator:
"""Returns total of metrics
"""
since, until = self.rt.since, datetime.now()
metrics = self.rt.extract()
metrics = self.rt.total
if self.history:
since, _, _ = self.history[0]
for _, _, c in self.history:
......@@ -169,6 +172,11 @@ class MetricsAggregator:
return metrics
def add(self, log):
"""
Parameters:
log (CommonLog): log to add
"""
assert isinstance(log, CommonLog), "expected log.CommonLog"
section = self.extract_section(log)
client = self.extract_client(log)
method = self.extract_method(log)
......
......@@ -7,19 +7,12 @@ class Task:
self.loop = loop or asyncio.get_event_loop()
self.task = None
self.running = False
self.debug = True
def start(self):
self.running = True
async def bubble():
try:
await self.run()
except KeyboardInterrupt:
pass
if not self.task or self.task.done():
coro = self.run() if self.debug else bubble()
coro = self.run()
self.task = self.loop.create_task(coro)
return self.task
......
from functools import singledispatch
def switch(a, b, reverse=False):
if reverse and b is not None:
def swap(a, b, switch=True):
if switch and b is not None:
a, b = b, a
return a, b
@singledispatch
def merge(a, b=None, reverse=False):
a, b = switch(a, b, reverse)
a, b = swap(a, b, reverse)
return b
@merge.register(dict)
def merge_dict(a, b=None, reverse=False):
a, b = switch(a, b, reverse)
a, b = swap(a, b, reverse)
dest = {}
dest.update(a)
if isinstance(b, dict):
......@@ -29,7 +29,7 @@ def merge_dict(a, b=None, reverse=False):
@merge.register(int)
@merge.register(float)
def merge_numbers(a, b=None, reverse=False):
a, b = switch(a, b, reverse)
a, b = swap(a, b, reverse)
dest = a
if isinstance(b, (int, float)):
dest += b
......@@ -40,7 +40,7 @@ def merge_numbers(a, b=None, reverse=False):
@merge.register(set)
def merge_set(a, b=None, reverse=False):
a, b = switch(a, b, reverse)
a, b = swap(a, b, reverse)
dest = set(a)
if isinstance(b, set):
dest.update(b)
......@@ -51,7 +51,7 @@ def merge_set(a, b=None, reverse=False):
@merge.register(list)
def merge_list(a, b=None, reverse=False):
a, b = switch(a, b, reverse)
a, b = swap(a, b, reverse)
dest = []
dest.extend(a)
if isinstance(b, (list, tuple)):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment