Skip to content

Commit

Permalink
Get real collectors
Browse files Browse the repository at this point in the history
  • Loading branch information
@joente
joente committed May 12, 2023
1 parent 2c45589 commit abd3af3
Show file tree
Hide file tree
Showing 2 changed files with 98 additions and 155 deletions.
251 changes: 97 additions & 154 deletions bin/infrasonar
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ from dataclasses import dataclass
from setproctitle import setproctitle
from typing import Any, Optional, Dict, List, Tuple

__version__ = '0.1.8' # Update version in setup as well
__version__ = '0.1.9' # Update version in setup as well


_labels_example = """
Expand Down Expand Up @@ -70,10 +70,11 @@ _kinds = (
'Apple',
'Azure',
'Citrix',
'Dell',
'DNS',
'Dell',
'Docker',
'Eaton',
'Email',
'Firewall',
'FreeBSD',
'HP',
Expand Down Expand Up @@ -146,179 +147,107 @@ _asset_keys = {
'collectors'
}
_collector_keys = set(['key', 'config'])

_spaces = re.compile(r'\s+')
_uri = re.compile(r'https?:\/\/.+')
_extract_name = re.compile(r'\w+')


def _test_use(o: Any, key: str, prop: str):
if not isinstance(o, str) or _spaces.search(o):
sys.exit(
f'Option "_use" ({key}) must be a string without '
'whitespace or null')

def _test_bool(o: Any, key: str, prop: str):
if not isinstance(o, bool):
sys.exit(f'Option "{prop}" ({key}) must be a boolean value')

def _test_name_servers(o: Any, key: str, prop: str):
if not isinstance(o, list) or \
not len(o) or \
not all([
isinstance(obj, str) and not _spaces.search(obj)
for obj in o]) or \
len(set(o)) != len(o):
sys.exit(
f'Option "nameServers" ({key}) is required and must be a list '
'with unique and at least one nameserver')

def _test_int(o: Any, key: str, prop: str):
if not isinstance(o, int):
sys.exit(f'Option "{prop}" ({key}) must be a integer value')

def _test_fqdn(o: Any, key: str, prop: str):
if not isinstance(o, str):
sys.exit(f'Option "fqdn" ({key}) must be a string or null')

def _test_float(o: Any, key: str, prop: str):
if not isinstance(o, float):
sys.exit(f'Option "{prop}" ({key}) must be a floating point value')

def _test_address(o: Any, key: str, prop: str):
if not isinstance(o, str) or _spaces.search(o):
sys.exit(
f'Option "address" ({key}) must be a string without '
'whitespace or null')


def _test_interval(o: Any, key: str, prop: str):
if not isinstance(o, int) or not (1 <= o <= 9):
sys.exit(
f'Option "interval" ({key}) must be an interger value '
'between 1 and 9')
def _test_string(o: Any, key: str, prop: str):
if not isinstance(o, str):
sys.exit(f'Option "{prop}" ({key}) must be a string value')


def _test_count(o: Any, key: str, prop: str):
if not isinstance(o, int) or not (1 <= o <= 9):
def _test_listbool(o: Any, key: str, prop: str):
if not isinstance(o, list) or \
not all([isinstance(obj, bool) for obj in o]):
sys.exit(
f'Option "count" ({key}) must be an interger value '
'between 1 and 9')
f'Option "{prop}" ({key}) is required and must be '
'a list with boolean values')


def _test_timeout(o: Any, key: str, prop: str):
if not isinstance(o, (float, int)) or not (0.0 <= o <= 240.0):
def _test_listint(o: Any, key: str, prop: str):
if not isinstance(o, list) or \
not all([isinstance(obj, int) for obj in o]) or \
len(set(o)) != len(o):
sys.exit(
f'Option "timeout" ({key}) must be a float value '
'between 0.0 and 240.0')
f'Option "{prop}" ({key}) is required and must be '
'a list with unique interger values')


def _test_ports(o: Any, key: str, prop: str):
def _test_listfloat(o: Any, key: str, prop: str):
if not isinstance(o, list) or \
not all([
isinstance(obj, int) and (0 < obj <= 65535)
for obj in o]) or \
not all([isinstance(obj, float) for obj in o]) or \
len(set(o)) != len(o):
sys.exit(
f'Option "{prop}" ({key}) is required and must be '
'a list with unique port numbers')
'a list with unique floating point values')


def _test_port(o: Any, key: str, prop: str):
if not isinstance(o, int) or not (0 < o <= 65535):
def _test_liststring(o: Any, key: str, prop: str):
if not isinstance(o, list) or \
not all([isinstance(obj, str) for obj in o]) or \
len(set(o)) != len(o):
sys.exit(
f'Option "port" ({key}) must be an interger value '
'between 1 and 65535')
f'Option "{prop}" ({key}) is required and must be '
'a list with unique string values')


_TYPE_MAP = {
'Bool': _test_bool,
'Int': _test_int,
'Float': _test_float,
'String': _test_string,
'ListBool': _test_listbool,
'ListInt': _test_listint,
'ListFloat': _test_listfloat,
'ListString': _test_liststring,
}


def _test_uri(o: Any, key: str, prop: str):
if not isinstance(o, str) or not _uri.match(o):
sys.exit(
f'Option "uri" ({key}) must be a valid URI')
async def aget_collectors(api: str, token: str, container_id: int,
verify_ssl: bool):
args = '?field=key&options=key,type,default'

url = _join(api, f'container/{container_id}/collectors{args}')
async with ClientSession(headers=_headers(token)) as session:
async with session.get(url, ssl=verify_ssl) as r:
if r.status != 200:
msg = await r.text()
raise Exception(f'{msg} (error code: {r.status})')
resp = await r.json()
return resp

def _test_bool(o: Any, key: str, prop: str):
if not isinstance(o, bool):
sys.exit(f'Option "{prop}" ({key}) must be a boolean value')

_collectors = {}

def _test_string(o: Any, key: str, prop: str):
if not isinstance(o, str):
sys.exit(f'Option "{prop}" ({key}) must be a string value')

def _load_collectos(api: str, token: str, container_id: int,
verify_ssl: bool):
"""Returns {collectorkey: {option: [verify_func, default], ..}, ..}
"""
global _collectors

_collectors = {
'dns': {
'nameServers': (_test_name_servers, []),
'fqdn': (_test_fqdn, ''),
},
'docker': None,
'eaton': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
},
'esx': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
},
'hpilo': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
},
'hpprocurve': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
},
'http': {
'uri': (_test_uri, ''),
'timeout': (_test_timeout, 10.0),
'verifySSL': (_test_bool, False),
'withPayload': (_test_bool, False),
'allowRedirects': (_test_bool, False),
},
'lastseen': None,
'mssql': {
'address': (_test_address, ''),
'port': (_test_port, 1433),
'_use': (_test_use, ''),
},
'netapp': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
},
'ping': {
'address': (_test_address, ''),
'interval': (_test_interval, 1),
'count': (_test_count, 5),
'timeout': (_test_timeout, 10.0),
},
'platform': None,
'santricity': {
'address': (_test_address, ''),
'port': (_test_port, 8443),
'storageSystemId': (_test_string, '1'),
'_use': (_test_use, ''),
},
'snmp': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
},
'synology': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
},
'tcp': {
'address': (_test_address, ''),
'checkCertificatePorts': (
_test_ports,
[443, 995, 993, 465, 3389, 989, 990, 636, 5986]
),
'checkPorts': (_test_ports, []),
},
'unifi': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
},
'vcenter': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
},
'wmi': {
'address': (_test_address, ''),
'_use': (_test_use, ''),
data = asyncio.run(aget_collectors(api, token, container_id, verify_ssl))
_collectors = {
c['key']: {
o['key']: (_TYPE_MAP[o['type']], o['default'])
for o in c['options']
} if c['options'] else None
for c in data
}
}


def check_collector(collector: dict, configs: dict,
Expand Down Expand Up @@ -371,8 +300,7 @@ def check_collector(collector: dict, configs: dict,
func(config.get(k, dval), key, k)


def check_asset(asset: dict, labels: dict, configs: dict,
allow_unknown_collectors: bool, allow_unknown_kinds: bool):
def check_asset(asset: dict, labels: dict, allow_unknown_kinds: bool):
too_much = set(asset.keys()) - _asset_keys
if too_much:
sys.exit(f'Unexpected key in asset: "{too_much.pop()}"')
Expand Down Expand Up @@ -418,6 +346,9 @@ def check_asset(asset: dict, labels: dict, configs: dict,
sys.exit(f'Asset label "{label}" missing in labels')
asset_labels[idx] = labels[label]


def check_asset_collectors(asset: dict, configs: dict,
allow_unknown_collectors: bool):
collectors = asset.get('collectors')
if collectors is not None:
if not isinstance(collectors, list) or \
Expand All @@ -428,7 +359,7 @@ def check_asset(asset: dict, labels: dict, configs: dict,


def sanity_check(data: dict, allow_unknown_collectors: bool,
allow_unknown_kinds: bool):
allow_unknown_kinds: bool, api: str, verify_ssl: bool):
too_much = set(data.keys()) - _yaml_keys
if too_much:
sys.exit(f'Unexpected key in yaml: "{too_much.pop()}"')
Expand Down Expand Up @@ -462,12 +393,7 @@ def sanity_check(data: dict, allow_unknown_collectors: bool,
sys.exit(_assets_example)

for asset in assets:
check_asset(
asset,
labels,
configs,
allow_unknown_collectors,
allow_unknown_kinds)
check_asset(asset, labels, allow_unknown_kinds)

if token is None:
try:
Expand All @@ -482,6 +408,10 @@ def sanity_check(data: dict, allow_unknown_collectors: bool,
elif not isinstance(token, str):
sys.exit('token must be a string or null')

_load_collectos(api, data['token'], container_id, verify_ssl)
for asset in assets:
check_asset_collectors(asset, configs, allow_unknown_collectors)


def collector_by_key(asset: dict, key: str):
collectors = asset.get('collectors')
Expand Down Expand Up @@ -767,7 +697,12 @@ def apply_assets(filename: str, api: str, verify_ssl: bool, add_only: bool,
if not isinstance(data, dict):
sys.exit('Expecting the yaml to conain a dict')

sanity_check(data, allow_unknown_collectors, allow_unknown_kinds)
sanity_check(
data,
allow_unknown_collectors,
allow_unknown_kinds,
api,
verify_ssl)

labels = data.get('labels', {})
container_id = data.get('container', 0)
Expand Down Expand Up @@ -943,8 +878,9 @@ def mod_assets_res(assets: list, labels: dict, include_defaults: bool):
control = _collectors.get(c['key'])
if control is not None:
for k in tuple(config.keys()):
if config[k] == control[k][1]:
del config[k]
if k in control:
if config[k] == control[k][1]:
del config[k]
if config:
c['config'] = config
cc.append(c)
Expand Down Expand Up @@ -989,6 +925,9 @@ def get_assets(container_id: int, api: str, verify_ssl: bool, output: str,
if label_ids:
labels = asyncio.run(async_get_labels(
api, token, verify_ssl, label_ids))

_load_collectos(api, token, container_id, verify_ssl)

mod_assets_res(assets, labels, include_defaults)

data = OrderedDict()
Expand Down Expand Up @@ -1034,12 +973,16 @@ def get_asset(asset_id: int, api: str, verify_ssl: bool, output: str,
assets = asyncio.run(aget_asset(
api, token, asset_id, verify_ssl, fields))
container_id = assets[0]['container']

labels = {}
if with_labels:
label_ids = get_label_ids(assets)
if label_ids:
labels = asyncio.run(async_get_labels(
api, token, verify_ssl, label_ids))

_load_collectos(api, token, container_id, verify_ssl)

mod_assets_res(assets, labels, include_defaults)

data = OrderedDict()
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

setup(
name='infrasonar',
version='0.1.8', # Update version in infrasonar as well
version='0.1.9', # Update version in infrasonar as well
description='InfraSonar Toolkit',
url='https://github.com/infrasonar/toolkit',
long_description=long_description,
Expand Down

0 comments on commit abd3af3

Please sign in to comment.