Introduction
The Audit Trail feature stores its log records as files in the filespace.
With the current version of the feature, there is no automatic mechanism to delete files based on certain criteria (such as files older than a year), a filespace administrator must manually remove irrelevant logs when needed.
A description of the audit log file hierarchy can be found here.
Deleting the logs manually
The log directory is protected, so you need to log in with a filespace admin user.
- Navigate to the .lucid_audit directory.
- Ensure that the logs you wish to delete are either:
-
- Ingested into another system, or
- Old/irrelevant and no longer needed.
-
- For each node ID, navigate to the directory/files you want to remove and delete the files.
Deleting the logs via our script
This script is provided for convenience. Please execute it with caution, since passing incorrect parameters can result in the loss of important security data.
Examples
The Python 3 script below can be used to delete records older than a given number of days. Here are some sample executions:
Delete logs older than 30 days if only a single filespace is connected:
python3 classic-delete-at-logs.py --days=30
Delete logs older than 60 days for a given filespace if there are multiple connections
python3 classic-delete-at-logs.py --filespace name.domain --days=60
The usage of the script is available by using the -h or --help option:
Usage: classic-delete-at-logs.py [-h] --days <number> [--filespace <filespace.domain>] [--lucid <path-to-executable>]
Options:
-h, --help show this help message and exit
--days <number>, -d <number>
Logs older than the specified days will be deleted.
--filespace <filespace.domain>, -f <filespace.domain>
Filespace whose logs will be deleted, can be omitted if only one fs is connected.
--lucid <path-to-executable>, -l <path-to-executable>
Path to the Lucid executable (e.g. /Applications/Lucid.app/Contents/Resources/Lucid or "C:\Program Files\Lucid\resources\Lucid.exe")
Scripts
The script is available for download at the bottom of the page.
Alternatively, you can copy the contents of the "Python" block below and create your own file:
-
'''
AUTHOR: [LucidLink Developer Team]
NAME: classic-delete-at-logs.py
Version: 1.0
DESCRIPTION: This script can be used to delete log files from the .lucid_audit folder.
It can be instrumented to delete logs older than X days.
THE SCRIPT IS PROVIDED “AS IS” AND “AS AVAILABLE” AND IS WITHOUT
WARRANTY OF ANY KIND. PLEASE REVIEW ALL TERMS AND CONDITIONS.
https://www.lucidlink.com/legal-documents'''
LucidLink Classic
This script can be used to delete log files from the .lucid_audit folder.
It can be instrumented to delete logs older than X days.
'''
import argparse
import datetime
import json
import subprocess
import sys
import os
AUDIT_FOLDER = '.lucid_audit'
TIMESTAMP_KEY = 'timestamp'
def main():
try:
lucid2_path = args.lucid if args.lucid else 'lucid2'
exit_if_no_lucid2(lucid2_path)
mount_point = get_mount_point(lucid2_path)
delete_audit_logs(mount_point)
except KeyboardInterrupt:
sys.exit(0)
def get_mount_point(lucid2_path: str) -> str:
print('Looking for mount point...')
status_cmd = [lucid2_path, 'status']
if args.filespace:
status_cmd = [status_cmd[0], '--name', args.filespace, status_cmd[1]]
status_lines = list(filter(len, get_output_or_exit_with_msg(status_cmd).splitlines()))
if not status_lines:
exit_no_mountpoint()
if "currently not running" in status_lines[0]:
print(status_lines[0])
sys.exit(5)
if status_lines[0].startswith("Multiple"):
print("Multiple filespaces connected. Specify one with --filespace <filespace.domain>")
list_filespaces = get_output_or_exit_with_msg([lucid2_path, 'list']).splitlines()
for line in list_filespaces:
if not line:
print()
words = line.split()
if len(words) > 1 and words[1] != 'ID':
print('\t{}'.format(words[1]))
sys.exit(2)
if status_lines[0].startswith("Lucid is currently not running for the specified name"):
exit_filespace_not_connected()
mount_point = None
for status_line in status_lines:
if not args.filespace and status_line.startswith('Filespace name: '):
args.filespace = status_line.removeprefix('Filespace name: ')
if status_line.startswith('Mount point: '):
mount_point = status_line.removeprefix('Mount point: ')
if mount_point is None:
exit_no_mountpoint()
return mount_point
def delete_audit_logs(mount_point: str):
print('Looking for audit logs...')
audit_dir = os.path.join(mount_point, AUDIT_FOLDER)
args.audit_dir = audit_dir
if not os.path.exists(audit_dir):
exit_no_logs()
now = datetime.datetime.now(datetime.timezone.utc)
delta = datetime.timedelta(days=args.days)
target_time = now - delta
args.target_time = target_time
delete_files_before(audit_dir, target_time)
################################################################
############################ Utils #############################
################################################################
def microsec_to_sec(microsec: int):
return microsec / 1000000
def time_formatted(datetime: datetime.datetime) -> str:
return datetime.strftime('%Y-%m-%d %H:%M:%S UTC')
def delete_files(file_paths: list[str]):
cnt = 0
for file_path in file_paths:
try:
os.remove(file_path)
cnt += 1
except Exception as e:
print(file_path, ': ', e)
continue
delete_subdirs_if_empty(args.audit_dir)
print('Deleted {} log file(s).'.format(cnt))
def delete_subdirs_if_empty(dir: str):
for root_dir, dirs, files in os.walk(dir, topdown=False):
if root_dir == dir:
break
try:
if not files and not dirs:
os.rmdir(root_dir)
except Exception as e:
print('Error deleting empty directory: {}'.format(e))
def file_contains_logs_after(filepath: os.path, time: datetime.datetime):
try:
with open(filepath) as log_file:
lines = log_file.read().splitlines()
if not lines:
return False # empty log file
json_log = None
last_line = lines.pop()
try:
json_log = json.loads(last_line)
except Exception as e:
json_err_str = str(e).replace('line 1', 'line {}'.format(len(lines) + 1))
raise Exception('Audit log file corrupted: {}'.format(json_err_str))
if TIMESTAMP_KEY not in json_log:
raise Exception('Wrong audit schema format, could not find {}'.format(TIMESTAMP_KEY))
last_log_time = datetime.datetime.fromtimestamp(microsec_to_sec(int(json_log[TIMESTAMP_KEY])),
datetime.timezone.utc)
return last_log_time >= time
except Exception as e:
print('{}: {}'.format(e, filepath))
return True
def delete_files_before(dir, time: datetime.datetime):
print('Reading audit logs...')
file_paths = []
for dirpath, _, files in os.walk(dir):
for filename in files:
if filename.endswith('active'):
continue
file_path = os.path.join(dirpath, filename)
if file_contains_logs_after(file_path, time):
continue
file_paths.append(file_path)
if len(file_paths) == 0:
exit_no_old_logs_found()
confirm_delete_logs(file_paths)
def confirm_delete_logs(file_paths: list[str]):
days_msg = ''
if args.days == 0:
days_msg = '{} log file(s)'.format(len(file_paths))
else:
days_str = days_str = 'day' if args.days == 1 else 'days'
days_msg = '{} log file(s) older than {} {} ({})'.format(len(file_paths), args.days, days_str,
time_formatted(args.target_time))
msg = "Delete all {} from {}?\nDo you want to continue? [Y]es [N]o".format(days_msg, args.filespace)
while True:
print(msg)
user_input = input().strip().lower()
if user_input.startswith('y'):
delete_files(file_paths)
break
if user_input.startswith('n'):
print("Operation cancelled. No log files deleted.")
break
print('Invalid choice.')
################################################################
###################### Prerequisite checks #####################
################################################################
def exit_if_no_lucid2(lucid2_path: str):
print('Looking for lucid2...')
res = get_output([lucid2_path, 'info'])
if isinstance(res, OSError) or not res.startswith('Lucid is currently not running') and not res.startswith(
'Operating sys') and not res.startswith("Multiple Lucid"):
exit_not_a_lucid_app()
def exit_negative_days():
print('Negative --days value given. Use non-negative, eg.: --days 30')
sys.exit()
def exit_no_logs():
print('No audit logs were found in the filespace {}'.format(args.filespace))
sys.exit()
def exit_not_a_lucid_app():
if args.lucid:
print('The given path is not to a valid lucid2 executable: {}'.format(args.lucid))
else:
print('Could not find the lucid2 executable, try specifying its path `--lucid <path-to-executable>`')
sys.exit()
def exit_no_mountpoint():
print('Could not find the mountpoint of the filespace {}'.format(args.filespace if args.filespace else ''))
sys.exit(3)
def exit_filespace_not_connected():
print('Filespace {} is currently not connected.'.format(args.filespace))
sys.exit(4)
def exit_no_old_logs_found():
if args.days > 0:
days_str = 'day' if args.days == 1 else 'days'
print('No log files older than {} {} ({}) were found in {}.'.format(args.days, days_str,
time_formatted(args.target_time),
args.filespace))
else:
print('No log files were found in {}.'.format(args.filespace))
sys.exit()
################################################################
#################### Command output parsing ####################
################################################################
def get_output_or_exit_with_msg(args) -> str:
res = get_output(args)
if isinstance(res, OSError):
print(res)
sys.exit(res.errno)
if res is None:
print('An unknown error occured.')
sys.exit(1)
return res
def get_output(args):
output = None
try:
res = subprocess.run(args, check=False, capture_output=True)
output = res.stdout if res.stdout else res.stderr
if not output:
return None
output = output.decode()
except OSError as e:
return e
return output
################################################################
###################### Arguments parsing #######################
################################################################
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--days', '-d', metavar='<number>', required=True, type=int, default=0,
help='Logs older than the specified days will be deleted.')
parser.add_argument('--filespace', '-f', metavar='<filespace.domain>', required=False,
help='Filespace whose logs will be deleted, can be omitted if only one fs is connected.')
parser.add_argument('--lucid', '-l', metavar='<path-to-executable>', required=False,
help='Path to the Lucid executable (e.g. /Applications/lucid.app/Contents/Resources/lucid or "C:\\Program Files\\lucid\\resources\\lucid.exe")')
args = parser.parse_args()
if args.days < 0:
exit_negative_days()
main()
The script is provided "as is" and "as available", and is without warranty of any kind. Please review all terms and conditions: https://www.lucidlink.com/legal-documents
Minimum required Python version: 3.9.0
Related to