#!/usr/bin/env python3 """ Script to extract exceptions from log files containing a specific string. Supports both individual files and directories. Generates an interactive HTML report with grouped exceptions. """ import re import sys import argparse from pathlib import Path from collections import defaultdict from datetime import datetime import html import json import urllib.parse def is_log_line(line): """Check if line matches the standard log format.""" return bool(re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \|', line)) def is_error_line(line): """Check if line is an ERROR level log line.""" return ' | ERROR | ' in line and is_log_line(line) def is_java_exception_line(line): """ Check if line is a Java exception (e.g., java.lang.NullPointerException). This typically follows an ERROR line. """ # Common Java exception patterns java_exception_patterns = [ r'^[a-zA-Z][a-zA-Z0-9_.$]*Exception', # Any exception class r'^[a-zA-Z][a-zA-Z0-9_.$]*Error', # Any error class r'^java\.', # Java standard exceptions r'^android\.', # Android exceptions r'^kotlin\.', # Kotlin exceptions ] stripped = line.strip() return any(re.match(pattern, stripped) for pattern in java_exception_patterns) def is_stack_trace_line(line): """ Check if line is a stack trace line. Typically starts with 'at ' or contains method calls. """ stripped = line.strip() return ( stripped.startswith('at ') or stripped.startswith('... ') or re.match(r'^\s*at\s+[\w.$]+\.[\w$]+\(', line) or 'more' in stripped.lower() and re.search(r'\d+\s+more', stripped) ) def extract_exception_key(exception_lines): """ Extract a key from exception lines to group similar exceptions. Uses the Java exception class name and first line of stack trace. Args: exception_lines: List of lines forming the exception Returns: String key for grouping """ if not exception_lines: return "Unknown Error" # Look for the Java exception class name exception_class = None exception_message = "" first_stack_line = None for line in exception_lines: stripped = line.strip() # Find the exception class (e.g., java.lang.NullPointerException) if not exception_class and is_java_exception_line(line): # Extract just the exception class and message parts = stripped.split(':', 1) exception_class = parts[0].strip() if len(parts) > 1: exception_message = parts[1].strip() # Truncate long messages if len(exception_message) > 80: exception_message = exception_message[:80] + "..." # Get the first stack trace line to add context if not first_stack_line and is_stack_trace_line(line): # Extract just the method/class info at_match = re.search(r'at\s+([\w.$]+\.[\w$]+)', stripped) if at_match: first_stack_line = at_match.group(1) # Build the key if exception_class: key = exception_class if exception_message: key = f"{exception_class}: {exception_message}" elif first_stack_line: key = f"{exception_class} at {first_stack_line}" return key # Fallback to ERROR message if no Java exception found for line in exception_lines: if is_error_line(line): match = re.search(r'\| ERROR \| (.+)$', line) if match: message = match.group(1).strip() if len(message) > 150: message = message[:150] + "..." return message return "Unknown Error" def extract_exceptions(log_file, search_string): """ Extract all Java stack trace exceptions from log file where the file contains the search string. Only captures exceptions that have actual Java stack traces. Args: log_file: Path to the log file search_string: String to search for in the file Returns: List of tuples (exception_lines, exception_key, line_number) """ try: with open(log_file, 'r', encoding='utf-8', errors='ignore') as f: content = f.read() except Exception as e: print(f"Warning: Could not read {log_file}: {e}") return [] # Check if search string exists in file if search_string not in content: return [] lines = content.splitlines() exceptions = [] current_exception = [] exception_start_line = 0 in_exception = False has_java_exception = False for line_num, line in enumerate(lines, 1): if is_error_line(line): # Save previous exception if it has a Java stack trace if current_exception and has_java_exception: key = extract_exception_key(current_exception) exceptions.append((current_exception, key, exception_start_line)) # Start of a new potential exception current_exception = [line] exception_start_line = line_num in_exception = True has_java_exception = False elif in_exception: # Check if this is a Java exception or stack trace if is_java_exception_line(line): has_java_exception = True current_exception.append(line) elif is_stack_trace_line(line): has_java_exception = True current_exception.append(line) elif line.strip() and (line.startswith('\t') or line.startswith(' ')): # Indented line, might be part of exception current_exception.append(line) elif line.strip().startswith('Caused by:'): # Chained exception has_java_exception = True current_exception.append(line) elif not is_log_line(line) and line.strip() and has_java_exception: # Non-log line that's part of the exception current_exception.append(line) else: # End of exception block if current_exception and has_java_exception: key = extract_exception_key(current_exception) exceptions.append((current_exception, key, exception_start_line)) current_exception = [] in_exception = False has_java_exception = False # Don't forget the last exception if file ends with one if current_exception and has_java_exception: key = extract_exception_key(current_exception) exceptions.append((current_exception, key, exception_start_line)) return exceptions def get_log_files(path): """ Get all log files from a path (file or directory). Args: path: Path object to file or directory Returns: List of Path objects for log files """ if path.is_file(): return [path] elif path.is_dir(): # Common log file extensions log_extensions = ['.log', '.txt', '.out'] log_files = [] # Recursively search for log files for ext in log_extensions: log_files.extend(path.rglob(f'*{ext}')) # Also include files without extension that might be logs for file in path.rglob('*'): if file.is_file() and not file.suffix and file not in log_files: log_files.append(file) return sorted(log_files) else: return [] def group_exceptions(files_with_exceptions): """ Group exceptions by their error message across all files. Args: files_with_exceptions: List of tuples (log_file, exceptions) where exceptions is a list of (exception_lines, key, line_number) Returns: Dictionary mapping exception keys to list of (log_file, exception_lines, line_number) """ grouped = defaultdict(list) for log_file, exceptions in files_with_exceptions: for exception_lines, key, line_number in exceptions: grouped[key].append((log_file, exception_lines, line_number)) return grouped def generate_html_report(grouped_exceptions, search_string, output_file, log_path): """ Generate an interactive HTML report with grouped exceptions. Args: grouped_exceptions: Dictionary from group_exceptions() search_string: The search string used output_file: Path to output HTML file log_path: Path to the log directory/file being analyzed """ path = log_path # Sort groups by number of occurrences (descending) sorted_groups = sorted( grouped_exceptions.items(), key=lambda x: len(x[1]), reverse=True ) html_content = f""" Java Exception Report - {html.escape(search_string)}

Java Exception Report

Search String: {html.escape(search_string)} | Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
Unique Exceptions: {len(sorted_groups)}
Total Occurrences: {sum(len(occurrences) for _, occurrences in sorted_groups)}
Files Affected: {len(set(log_file for _, occurrences in sorted_groups for log_file, _, _ in occurrences))}

Exception Groups ({len(sorted_groups)}/{len(sorted_groups)})

Select an exception to view details
""" # Add detail panels for idx, (key, occurrences) in enumerate(sorted_groups): files_affected = len(set(log_file for log_file, _, _ in occurrences)) html_content += f"""

{html.escape(key)}

{len(occurrences)} occurrence(s) across {files_affected} file(s)
""" for log_file, exception_lines, line_number in occurrences: # Syntax highlighting for stack trace highlighted_trace = "" for line in exception_lines: line_html = html.escape(line) # Skip the ERROR log line for cleaner display if is_error_line(line): continue highlighted_trace += line_html + "\\n" # Create file:line URL for opening in editor file_path_abs = str(log_file.absolute()) file_uri = f"file://{file_path_abs}" html_content += f"""
{html.escape(str(log_file.name))}
Line {line_number}
{highlighted_trace.rstrip()}
""" html_content += """
""" html_content += """

Loading...

Loading file...
""" # Write the HTML file with open(output_file, 'w', encoding='utf-8') as f: f.write(html_content) print(f"\nHTML report generated: {output_file}") print(f"Open in browser: file://{output_file.absolute()}") def main(): parser = argparse.ArgumentParser( description='Extract and analyze exceptions from log files containing a specific string.', formatter_class=argparse.RawDescriptionHelpFormatter, epilog=''' Examples: %(prog)s /path/to/app.log PINKY %(prog)s /path/to/logs PINKY -o report.html %(prog)s /path/to/logs "Connection timeout" --output-dir ./reports ''' ) parser.add_argument( 'path', nargs='?', default='/Users/matthewroberts/development/projects/somewear/logs/android', help='Path to log file or directory (default: %(default)s)' ) parser.add_argument( 'search_string', nargs='?', default='callsign=PARTS', help='String to search for in log files (default: %(default)s)' ) parser.add_argument( '-o', '--output', help='Output HTML file path (default: exceptions_report.html in current directory)', default='/Users/matthewroberts/development/projects/somewear/celilo/exceptions_report.html' ) parser.add_argument( '--output-dir', help='Directory to save the HTML report (default: current directory)' ) parser.add_argument( '--no-console', action='store_true', help='Skip console output, only generate HTML report' ) args = parser.parse_args() path = Path(args.path) if not path.exists(): print(f"Error: Path '{args.path}' not found") sys.exit(1) print(f"Searching for exceptions containing string '{args.search_string}'...") log_files = get_log_files(path) if not log_files: print(f"No log files found in '{args.path}'") sys.exit(1) print(f"Found {len(log_files)} log file(s) to search\n") total_exceptions = 0 files_with_exceptions = [] for log_file in log_files: exceptions = extract_exceptions(log_file, args.search_string) if exceptions: files_with_exceptions.append((log_file, exceptions)) total_exceptions += len(exceptions) if not files_with_exceptions: print(f"No exceptions found containing '{args.search_string}'") return print(f"Found {total_exceptions} exception(s) in {len(files_with_exceptions)} file(s)\n") # Group exceptions grouped = group_exceptions(files_with_exceptions) # Determine output file path if args.output: output_file = Path(args.output) else: output_dir = Path(args.output_dir) if args.output_dir else Path.cwd() output_file = output_dir / 'exceptions_report.html' # Ensure output directory exists output_file.parent.mkdir(parents=True, exist_ok=True) # Generate HTML report generate_html_report(grouped, args.search_string, output_file, path) # Console output (unless --no-console is specified) if not args.no_console: print("\n" + "=" * 80) print("CONSOLE SUMMARY") print("=" * 80) # Sort groups by occurrence count sorted_groups = sorted( grouped.items(), key=lambda x: len(x[1]), reverse=True ) for key, occurrences in sorted_groups: print(f"\n[{len(occurrences)} occurrences] {key}") print("-" * 80) # Show first occurrence as an example if occurrences: log_file, exception_lines, line_number = occurrences[0] print(f"Example from: {log_file}:{line_number}") for line in exception_lines[:10]: # Show first 10 lines print(line) if len(exception_lines) > 10: print(f"... ({len(exception_lines) - 10} more lines)") # List all affected files affected_files = set(log_file for log_file, _, _ in occurrences) if len(affected_files) > 1: print(f"\nAlso found in {len(affected_files) - 1} other file(s):") for log_file in list(affected_files)[1:6]: # Show up to 5 more files print(f" - {log_file}") if len(affected_files) > 6: print(f" ... and {len(affected_files) - 6} more") print("\n" + "=" * 80) if __name__ == "__main__": main()