feat: Refactor log-alert application into modular structure

- Introduced alerters package with base Alerter class and implementations for LogAlerter and GotifyAlerter.
- Created fetchers package with abstract LogFetcher class and implementations for FileLogFetcher, LokiLogFetcher, and ParseableLogFetcher.
- Added filters package with base Filter class and implementations for RegexpFilter and GeolocationFilter.
- Implemented rules package with base AlertRule class and SimpleAlertRule for alerting logic.
- Enhanced configuration handling with utilities for loading and validating JSON configuration.
- Updated logging setup for better logging management.
- Modified main application logic to utilize the new modular structure, improving maintainability and readability.
- Updated config.json and config.schema.json to reflect changes in alerting and fetching configurations.
- Added python-dateutil to requirements for date parsing functionality.

log-alert/fetchers/__init__.py

@@ -0,0 +1,8 @@
+"""Fetchers package for log-alert."""
+
+from .base import LogFetcher
+from .file import FileLogFetcher
+from .loki import LokiLogFetcher
+from .parseable import ParseableLogFetcher
+
+__all__ = ["LogFetcher", "FileLogFetcher", "LokiLogFetcher", "ParseableLogFetcher"]

log-alert/fetchers/base.py

@@ -0,0 +1,36 @@
+"""Abstract base class for log fetchers."""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List
+
+
+class LogFetcher(ABC):
+    """Abstract base class for log fetchers."""
+
+    """Fetch logs."""
+    @abstractmethod
+    def fetch_logs(self, filters: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Fetch logs within the specified time range.
+        
+        Args:
+            filters: Dictionary of filters to apply
+            
+        Returns:
+            List of log entries
+        """
+        pass
+
+    """ Fetch logs with time range."""
+    @abstractmethod
+    def fetch_logs_time_range(self, filters: Dict[str, Any], start_time: int, end_time: int) -> List[Dict[str, Any]]:
+        """Fetch logs within the specified time range.
+        
+        Args:
+            filters: Dictionary of filters to apply
+            start_time: Start time in seconds since epoch
+            end_time: End time in seconds since epoch
+            
+        Returns:
+            List of log entries
+        """
+        pass

log-alert/fetchers/file.py

@@ -0,0 +1,81 @@
+"""File log fetcher implementation."""
+
+import logging
+import requests
+from typing import Dict, Any, List
+from dateutil.parser import parse
+
+from .base import LogFetcher
+
+logger = logging.getLogger("log-alert")
+
+
+class FileLogFetcher(LogFetcher):
+    """Concrete implementation for fetching logs from a file."""
+
+    type = "TAIL"
+
+    def __init__(self, config: Dict[str, Any]):
+        try:
+            self.file = open(config["file"], "r")
+        except FileNotFoundError:
+            logger.error(f"Log file not found: {config['file']}")
+            raise
+        self.position = 0  # To keep track of the last read position
+
+    def fetch_logs(self, filters: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Fetch logs from the file."""
+        logs = []
+
+        self.file.seek(self.position)  # Move to the last read position
+        while True:
+            self.position = self.file.tell()
+            line = self.file.readline()
+            if not line:
+                break  # End of file reached, exit loop
+
+            logger.debug(f"Read line: {line.strip()} at position {self.position}")
+
+            parts = line.strip().split(maxsplit=1)
+            timestamp_str, rest = parts[0], parts[1]
+
+            # Parse the timestamp string into a datetime object
+            dt = parse(timestamp_str)
+            # Convert to seconds since epoch
+            epoch_seconds = int(dt.timestamp())
+
+            log_entry = {"timestamp": epoch_seconds, "log": rest}
+
+            logs.append(log_entry)
+
+        return logs
+
+    def fetch_logs_time_range(self, filters: Dict[str, Any], start_time: int, end_time: int) -> List[Dict[str, Any]]:
+        """Fetch logs from the file within the specified time range."""
+        logs = []
+
+        self.file.seek(0)
+        while True:
+            line = self.file.readline()
+            if not line:
+                break  # End of file reached, exit loop
+
+            logger.debug(f"Read line: {line.strip()} at position {self.position}")
+
+            parts = line.strip().split(maxsplit=1)
+            timestamp_str, rest = parts[0], parts[1]
+
+            dt = parse(timestamp_str)
+            epoch_seconds = int(dt.timestamp())
+
+            if epoch_seconds < start_time:
+                continue
+            if epoch_seconds > end_time:
+                break
+
+            log_entry = {"timestamp": epoch_seconds, "log": rest}
+
+            logs.append(log_entry)
+
+        return logs
+    

log-alert/fetchers/loki.py

@@ -0,0 +1,60 @@
+"""Loki log fetcher implementation."""
+
+import logging
+from time import time
+import requests
+from typing import Dict, Any, List
+
+from .base import LogFetcher
+
+logger = logging.getLogger("log-alert")
+
+
+class LokiLogFetcher(LogFetcher):
+    """Concrete implementation for fetching logs from Loki."""
+
+    def __init__(self, config: Dict[str, Any]):
+        self.url = config["url"]
+        self.last_fetched_time = int(time.time())
+
+    def fetch_logs(self, filters: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Fetch logs from Loki without time range."""
+        old_time = self.last_fetched_time
+        self.last_fetched_time = int(time.time())
+        return self.fetch_logs_time_range(filters, old_time, self.last_fetched_time)
+
+    def fetch_logs_time_range(self, filters: Dict[str, Any], start_time: int, end_time: int) -> List[Dict[str, Any]]:
+        """Fetch logs from Loki within the specified time range."""
+        query = '{'
+        for label in filters.get("labels", {}):
+            if len(query) > 1:
+                query += ','
+            query += f'{label}="{filters["labels"][label]}"'
+        query += '}'
+        if "text" in filters:
+            query += f' |= "{filters["text"]}"'
+        logger.debug(f"Executing Loki query: {query}")
+        payload = {
+            "query": query,
+            "limit": 1000,
+            "start": str(int(start_time) * 1000000000),  # Convert to nanoseconds
+            "end": str(int(end_time) * 1000000000),
+            "direction": "forward"
+        }
+        try:
+            response = requests.get(f"{self.url}/loki/api/v1/query_range", params=payload)
+            response.raise_for_status()
+            data = response.json()
+            logs = []
+            for stream in data.get("data", {}).get("result", []):
+                for value in stream.get("values", []):
+                    timestamp, log = value
+                    logs.append({
+                        "timestamp": timestamp,
+                        "log": log,
+                        "labels": stream.get("stream", {})
+                    })
+            return logs
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Error fetching logs from Loki: {e}")
+            return []

log-alert/fetchers/parseable.py

@@ -0,0 +1,64 @@
+"""Parseable log fetcher implementation."""
+
+import datetime
+import logging
+from time import time
+import requests
+from typing import Dict, Any, List
+
+from .base import LogFetcher
+
+logger = logging.getLogger("log-alert")
+
+
+class ParseableLogFetcher(LogFetcher):
+    """Concrete implementation for fetching logs from Parseable."""
+
+    def __init__(self, config: Dict[str, Any]):
+        self.url = config["url"]
+        self.dataset = config["dataset"]
+        self.user = config.get("user")
+        self.password = config.get("password")
+        self.last_fetched_time = int(time())  # Initialize with current time
+
+    def fetch_logs(self, filters: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Fetch logs from Parseable without time range."""
+        old_time = self.last_fetched_time
+        self.last_fetched_time = int(time())
+        return self.fetch_logs_time_range(filters, old_time, self.last_fetched_time)
+
+    def fetch_logs_time_range(self, filters: Dict[str, Any], start_time: int, end_time: int) -> List[Dict[str, Any]]:
+        """Fetch logs from Parseable within the specified time range."""
+        query = 'SELECT * FROM \''
+        query += f'{self.dataset}\' WHERE '
+        labelNum = 0
+        for label in filters.get("labels", {}):
+            if labelNum > 0:
+                query += ' AND '
+            query += f'{label}=\'{filters["labels"][label]}\''
+            labelNum += 1
+        if "text" in filters:
+            if labelNum > 0:
+                query += ' AND '
+            query += f'log LIKE \'%{filters["text"]}%\''
+        logger.debug(f"Executing Parseable query: {query}")
+        payload = {
+            "query": query,
+            "startTime": datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "endTime": datetime.datetime.fromtimestamp(end_time).strftime("%Y-%m-%dT%H:%M:%SZ")
+        }
+        try:
+            response = requests.post(f"{self.url}/api/v1/query", json=payload, auth=(self.user, self.password), headers = { "Content-Type": "application/json" })
+            response.raise_for_status()
+            data = response.json()
+            logs = []
+            for item in data:
+                logs.append({
+                    "timestamp": item.get("p_timestamp"),
+                    "log": item.get("log"),
+                    "labels": {k: v for k, v in item.items() if k not in ["p_timestamp", "log"]}
+                })
+            return logs
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Error fetching logs from Parseable: {e}")
+            return []