Skip to content

Commit 6887d14

Browse files
authored
v.1.2.0
1 parent 163a490 commit 6887d14

27 files changed

+1382
-249
lines changed

Dockerfile

+9-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM python:3.9.6-slim-buster
1+
FROM python:3.10-slim-bullseye
22
LABEL MAINTAINER="Mike Schiessl - mike.schiessl@akamai.com"
33
LABEL APP_LONG="Akamai Universal Log Streamer"
44
LABEL APP_SHORT="ULS"
@@ -11,7 +11,7 @@ ARG ULS_DIR="$HOMEDIR/uls"
1111
ARG EXT_DIR="$ULS_DIR/ext"
1212

1313
ARG ETP_CLI_VERSION="0.3.5"
14-
ARG EAA_CLI_VERSION="0.4.2"
14+
ARG EAA_CLI_VERSION="0.4.4"
1515
ARG MFA_CLI_VERSION="0.0.6"
1616

1717
# ENV VARS
@@ -27,7 +27,8 @@ RUN apt-get update && \
2727
ca-certificates \
2828
git \
2929
curl \
30-
telnet && \
30+
telnet \
31+
gcc libssl-dev libffi-dev && \
3132
rm -rf /var/lib/apt/lists/
3233

3334
# USER & GROUP
@@ -42,22 +43,26 @@ RUN mkdir -p ${ULS_DIR}
4243
# Install ULS
4344
COPY bin/ ${ULS_DIR}/bin
4445
WORKDIR ${ULS_DIR}
46+
RUN pip3 install -r ${ULS_DIR}/bin/requirements.txt
4547

4648
# Install external CLI'S
4749
## ETP CLI
4850
ENV ETP_CLI_VERSION=$ETP_CLI_VERSION
4951
RUN git clone --depth 1 -b "${ETP_CLI_VERSION}" --single-branch https://github.com/akamai/cli-etp.git ${EXT_DIR}/cli-etp && \
5052
pip3 install -r ${EXT_DIR}/cli-etp/requirements.txt
53+
5154
## EAA CLI
5255
ENV EAA-CLI_VERSION=$EAA_CLI_VERSION
5356
RUN git clone --depth 1 -b "${EAA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-eaa.git ${EXT_DIR}/cli-eaa && \
5457
pip3 install -r ${EXT_DIR}/cli-eaa/requirements.txt
58+
59+
5560
## MFA CLI
5661
ENV MFA-CLI_VERSION=$MFA_CLI_VERSION
5762
RUN git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-mfa.git ${EXT_DIR}/cli-mfa && \
5863
pip3 install -r ${EXT_DIR}/cli-mfa/requirements.txt
5964

6065
# ENTRYPOINTS / CMD
61-
ENTRYPOINT ["/usr/local/bin/python3","bin/uls.py"]
66+
ENTRYPOINT ["/usr/local/bin/python3","-u","bin/uls.py"]
6267
#CMD ["--help"]
6368
# EOF

README.md

+8-5
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,13 @@ It can be run directly as Python code, as a provided Docker container or through
4646

4747

4848
- Supported data outputs
49-
- TCP Socket (tcp://host:port) `--output tcp`
50-
- UDP Socket (udp://host:port) `--output udp`
51-
- HTTP(S) URL (http(s)://host:port/path) (supporting Authentication) `--output http`
52-
- RAW (>STDOUT) `--output raw`
53-
49+
- [`TCP` Socket (tcp://host:port) `--output tcp`](docs/OUTPUTS.md#tcp--udp)
50+
- [`UDP` Socket (udp://host:port) `--output udp`](docs/OUTPUTS.md#tcp--udp)
51+
- [`HTTP` and `HTTPS` URL (http(s)://host:port/path) (supporting Authentication) `--output http`](docs/OUTPUTS.md#http-and-https)
52+
- [`RAW` (>STDOUT) `--output raw`](docs/OUTPUTS.md#raw)
53+
- [`FILE` (writes log to a file on the local filesystem and rotates them) `--output file`](docs/OUTPUTS.md#file)
54+
The output documentation can be found [here](docs/OUTPUTS.md).
55+
5456

5557
- Operation types
5658
- [python (command line)](./docs/COMMAND_LINE_USAGE.md)
@@ -64,6 +66,7 @@ It can be run directly as Python code, as a provided Docker container or through
6466
- HTTP CA CERT verification skipping
6567
- Adoptable HTTP - POST format
6668
- [Output filter](docs/ADDITIONAL_FEATURES.md#filter---filter-feature) (to reduce number of log lines sent to SIEM)
69+
- [Output transformation](docs/TRANSFORMATIONS.md) (to produce other log formats i.e. for 3rd party integrations)
6770

6871
## Documentation
6972
ULS can be operated in many ways.

bin/config/global_config.py

+24-7
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/env python3
22

33
# Common global variables / constants
4-
__version__ = "1.1.0"
4+
__version__ = "1.2.0"
55
__tool_name_long__ = "Akamai Unified Log Streamer"
66
__tool_name_short__ = "ULS"
77

@@ -11,6 +11,9 @@
1111
output_line_breaker = '\r\n' # Line breaking type (to split messages when streaming data)
1212
main_wait_default = 0.01 # Default wait time within the main loop
1313
main_wait_max = 60 # Maximum wait time for the main loop
14+
main_resend_attempts = 10 # Maximum number of attempts to deliver the data
15+
main_resend_exit_on_fail = False # Stop program, if a single logline was not able to be delivered after $main_resend_attempts
16+
1417

1518
# EAA
1619
# Path to the EAA CLI Executable
@@ -33,13 +36,16 @@
3336
input_format_choices = ['JSON', 'TEXT'] # Available input format choices (need to be supported by cli)
3437

3538
# OUTPUT Choices
36-
output_choices = ['TCP', 'HTTP', 'UDP', 'RAW'] # Definition of OUTPUT Choices
39+
output_choices = ['TCP', 'HTTP', 'UDP', 'RAW', 'FILE'] # Definition of OUTPUT Choices
40+
41+
# Transformation Choices
42+
transformation_choices = ['MCAS', 'JMESPATH']
43+
3744

3845
# LogLevels
3946
log_levels_available = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
4047
log_level_default = 'WARNING'
4148

42-
4349
# INPUT Configuration
4450
input_rerun_retries = 3 # Number of rerun attempts before giving up
4551
input_run_delay = 1 # Time in seconds to wait for the first health check
@@ -49,13 +55,23 @@
4955
# OUTPUT Configuration
5056
output_reconnect_retries = 10 # Number of reconnect attempts before giving up
5157
output_reconnect_delay = 1 # Time in seconds between reconnect attempts
58+
## TCP / UDP
5259
output_udp_send_buffer = 262144 # UDP Send buffer in bytes
5360
output_udp_timeout = 10.0 # UDP SEND / CONNECT Timeout (seconds)
5461
output_tcp_send_buffer = 262144 # TCP Send buffer in bytes
5562
output_tcp_timeout = 10.0 # TCP SEND / CONNECT Timeout (seconds)
56-
# Additional Headers to send (requests module KV pairs)
57-
output_http_header = {'User-Agent': f'{__tool_name_long__}/{__version__}'}
58-
63+
## HTTP
64+
output_http_header = {'User-Agent': f'{__tool_name_long__}/{__version__}'} # HTTP Additional Headers to send (requests module KV pairs)
65+
## FILE
66+
output_file_encoding = "utf-8" # FILE Encoding setting
67+
output_file_handler_choices = ['SIZE', 'TIME'] # Available Choices for the file handler
68+
output_file_default_name = 'tmp/uls_file.output' # Default file name (path + name), only used if not set
69+
output_file_default_backup_count = 3 # Default number of backup files (after rotation)
70+
output_file_default_maxbytes = 50* 1024 * 1024 # Default maximum size of a file when rotated by the FILE - handler
71+
output_file_default_time_use_utc = False # Use UTC instead of local system time (Default: False)
72+
output_file_time_choices = ['S','M','H','D','W0','W1','W2','W3','W4','W5','W6','MIDNIGHT'] # Available choices for the time unit
73+
output_file_time_default = 'M' # Default value for the time unit (Minutes)
74+
output_file_time_interval = 30 # Default value for the interval (30)
5975

6076
# Monitoring Configuration
6177
monitoring_enabled = True # Set to false to disable monitoring outputs
@@ -66,4 +82,5 @@
6682
edgerc_openapi = ["host", "client_token", "client_secret", "access_token"] # required fields for OPENAPI
6783
edgerc_eaa_legacy = ["eaa_api_host", "eaa_api_key", "eaa_api_secret"] # required for EAA - Legacy
6884
edgerc_mfa = ["mfa_integration_id", "mfa_signing_key"] # Required for MFA
69-
edgerc_documentation_url = "https://github.com/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md"
85+
edgerc_documentation_url = "https://github.com/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md"
86+
edgerc_mock_file = "ext/edgerc" # Required for display the version if no edgercfile was given

bin/config/transformation_config.py

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env python3
2+
3+
4+
# MCAS
5+
## Global
6+
mcas_name_long = "Microsoft Cloud Applican Security"
7+
mcas_name_short = "MCAS"
8+
9+
## Requirements
10+
mcas_required_input = ['ETP']
11+
mcas_required_feed = ['PROXY', 'DNS']
12+
mcas_required_cliformat = ['JSON']
13+
14+
## MCAS INPUT LOG FORMAT
15+
mcas_input_format = "detection_time={0} client_ip={1} destination_ip={2} domain={3} user_name={4} bytes_uploaded={5} bytes_downloaded={6} bytes_total={7} action={8}"
16+
17+
18+
# JMESPATH
19+
## Global
20+
jmespath_name_long = "JMESPath https://jmespath.org/"
21+
jmespath_name_short = "JMESPath"
22+
23+
## Requirements
24+
jmespath_required_cliformat = ['JSON']

bin/modules/UlsArgsParser.py

+110-16
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,9 @@ def init():
7070
dest='inproxy',
7171
type=str,
7272
default=(os.environ.get('ULS_INPUT_PROXY') or None),
73-
help=argparse.SUPPRESS)
74-
# We're surpressing this for now, as the param does not seem to work (mschiess-20210818 - see EME-498)
75-
#help="Use a proxy Server for the INPUT requests (fetching data from AKAMAI API'S)")
73+
help="Use a proxy Server for the INPUT requests (fetching data from AKAMAI API'S)")
74+
#help=argparse.SUPPRESS)
75+
7676
# RAWCMD
7777
input_group.add_argument('--rawcmd',
7878
action='store',
@@ -94,6 +94,21 @@ def init():
9494
default=(os.environ.get('ULS_SECTION') or 'default'),
9595
help="Credentials file Section's name to use ('default' if not specified).")
9696

97+
# Log Starttime
98+
input_group.add_argument('--starttime',
99+
action='store',
100+
type=int,
101+
dest="starttime",
102+
default=(os.environ.get('ULS_STARTTIME') or None),
103+
help="Start time (EPOCH SECONDS) from when to start getting logs ('default': cli_default (now), example: '1631556101')")
104+
# Log Endtime
105+
input_group.add_argument('--endtime',
106+
action='store',
107+
type=int,
108+
dest="endtime",
109+
default=(os.environ.get('ULS_ENDTIME') or None),
110+
help="End time (EPOCH SECONDS) until when to stop getting logs ('default': cli_default (never), example: '1631556101')")
111+
97112
# ----------------------
98113
# Output GROUP
99114
output_group = parser.add_argument_group(title="Output",
@@ -107,44 +122,39 @@ def init():
107122
choices=uls_config.output_choices,
108123
help="Select the Output Destination Default: None")
109124

110-
# Output HOST
125+
# TCP / UPD
126+
## Output HOST
111127
output_group.add_argument('--host',
112128
action='store',
113129
type=str,
114130
default=(os.environ.get('ULS_OUTPUT_HOST') or None),
115131
help="Host for TCP/UDP")
116132

117-
# OUTPUT PORT
133+
## OUTPUT PORT
118134
output_group.add_argument('--port',
119135
action='store',
120136
type=int,
121137
default=int(os.environ.get('ULS_OUTPUT_PORT') or '0'),
122138
help="Port for TCP/UDP")
123139

124-
# Output FILTER
125-
output_group.add_argument('--filter',
126-
action='store',
127-
type=str,
128-
default=(os.environ.get('ULS_OUTPUT_FILTER') or None),
129-
help="Filter (regex) to reduce number of sent log files (Only send lines that match the --filter argument).")
130-
131-
# HTTP URL
140+
# HTTP
141+
## HTTP URL
132142
output_group.add_argument('--httpurl',
133143
action='store',
134144
type=str,
135145
default=(os.environ.get('ULS_HTTP_URL') or None),
136146
help=f'Full http(s) target url i.e. '
137147
f'https://my.splunk.host:9091/services/collector/event"')
138148

139-
# HTTP AUTH HEADER
149+
## HTTP AUTH HEADER
140150
output_group.add_argument('--httpauthheader',
141151
action='store',
142152
type=str,
143153
default=(os.environ.get('ULS_HTTP_AUTH_HEADER') or None),
144154
help='HTTP Header for authorization. Example: '
145155
'\'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}\'')
146156

147-
# Disable HTTP TLS verification
157+
## Disable HTTP TLS verification
148158
output_group.add_argument('--httpinsecure',
149159
action='store',
150160
type=bool,
@@ -153,14 +163,98 @@ def init():
153163
const=True,
154164
help=f'Disable TLS CA Certificate verification. Default: False')
155165

156-
# HTTP FORMAT DEFINITION
166+
## HTTP FORMAT DEFINITION
157167
output_group.add_argument('--httpformat',
158168
action='store',
159169
type=str,
160170
default=(os.environ.get('ULS_HTTP_FORMAT') or '{"event": %s}'),
161171
help='HTTP Message format expected by http receiver '
162172
'(%%s defines the data string). Default \'{\"event\": %%s}\'')
163173

174+
# FILE STUFF
175+
## File Handler
176+
output_group.add_argument('--filehandler',
177+
action='store',
178+
type=str.upper,
179+
default=(os.environ.get('ULS_FILE_HANDLER') or None),
180+
choices=uls_config.output_file_handler_choices,
181+
help=f"Output file handler - Decides when files are rotated -"
182+
f"Choices: {uls_config.output_file_handler_choices} -"
183+
f" Default: None")
184+
## File Name
185+
output_group.add_argument('--filename',
186+
action='store',
187+
type=str,
188+
default=(os.environ.get('ULS_FILE_NAME') or
189+
uls_config.output_file_default_name),
190+
help=f"Output file destination (path + filename)"
191+
f" Default: {uls_config.output_file_default_name}")
192+
193+
## File Backup count
194+
output_group.add_argument('--filebackupcount',
195+
action='store',
196+
type=int,
197+
default=(os.environ.get('ULS_FILE_BACKUPCOUNT') or
198+
uls_config.output_file_default_backup_count),
199+
help=f"Number of rotated files to keep (backup)"
200+
f" Default: {uls_config.output_file_default_backup_count}")
201+
202+
## File Max bytes
203+
output_group.add_argument('--filemaxbytes',
204+
action='store',
205+
type=int,
206+
default=(os.environ.get('ULS_FILE_MAXBYTES') or
207+
uls_config.output_file_default_maxbytes),
208+
help=f"Number of rotated files to keep (backup)"
209+
f" Default: {uls_config.output_file_default_maxbytes} bytes")
210+
211+
## File Time
212+
output_group.add_argument('--filetime',
213+
action='store',
214+
type=str.upper,
215+
default=(os.environ.get('ULS_FILE_TIME') or
216+
uls_config.output_file_time_default),
217+
choices=uls_config.output_file_time_choices,
218+
help=f"Specifies the file rotation trigger unit "
219+
f" Default: {uls_config.output_file_time_default}, Valid Choices: {uls_config.output_file_time_choices}")
220+
221+
## File Time Interval
222+
output_group.add_argument('--fileinterval',
223+
action='store',
224+
type=int,
225+
default=(os.environ.get('ULS_FILE_INTERVAL') or
226+
uls_config.output_file_time_interval),
227+
help=f"Specifies the file rotation interval based on `--filetime` unit value"
228+
f" Default: {uls_config.output_file_time_interval}")
229+
230+
# ----------------------
231+
special_group = parser.add_argument_group(title="Transformation",
232+
description="Define Module Settings (Output manipulation)")
233+
234+
# Output FILTER
235+
special_group.add_argument('--filter',
236+
action='store',
237+
type=str,
238+
default=(os.environ.get('ULS_OUTPUT_FILTER') or None),
239+
help="Filter (regex) to reduce number of sent log files (Only send lines that match the --filter argument).")
240+
241+
# Transformation Handler
242+
special_group.add_argument('--transformation',
243+
action='store',
244+
dest="transformation",
245+
type=str.upper,
246+
default=(os.environ.get('ULS_TRANSFORMATION') or None),
247+
choices=uls_config.transformation_choices,
248+
help="Select a transformation to manipulate the output format (optional)")
249+
250+
special_group.add_argument('--transformpattern', '--transformationpattern',
251+
action='store',
252+
dest="transformationpattern",
253+
type=str,
254+
default=(os.environ.get('ULS_TRANSFORMATION_PATTERN') or None),
255+
help="Provide a pattern to transform the output (Required for JMESPATH)")
256+
164257
return parser.parse_args()
165258

259+
166260
# EOF

0 commit comments

Comments
 (0)