fix: Improve log fetching by filtering timestamps

log-alert/config.json

@@ -26,12 +26,19 @@
                 "name": "parseable",
                 "filters": {
                     "labels": {
-                        "container_name": "/openssh-server"
+                        "container_name": "openssh-server"
                     },
-                    "text": "Accepted"
+                    "text": "Accepted",
+                    "timestamp": "date"
                 }
             },
             "filters": [
+                {
+                    "type": "timestamp",
+                    "config": {
+                        "source-field": "date"
+                    }
+                },
                 {
                     "type": "regexp",
                     "config": {
@@ -47,7 +54,7 @@
             ],
             "alerter": {
                 "name": "gotify",
-                "title": "Outside SSH login",
+                "title": "SSH login from {country}",
                 "message": "New SSH login for {username} on {hostname} from ip {ip} (country: {country}, provider: {isp}, method: {method})"
             }
         },
@@ -64,6 +71,14 @@
                 }
             },
             "filters": [
+                {
+                    "type": "timestamp",
+                    "config": {
+                        "source-field": "timestamp",
+                        "timezone": "Europe/Paris",
+                        "timestamp-format": "%b %d %H:%M:%S"
+                    }
+                },
                 {
                     "type": "regexp",
                     "config": {

log-alert/config.schema.json

@@ -46,7 +46,7 @@
               "type": "object",
               "required": ["type", "config"],
               "properties": {
-                "type": { "type": "string", "enum": ["regexp", "geolocation"] },
+                "type": { "type": "string", "enum": ["geolocation", "regexp", "timestamp"] },
                 "config": {
                   "type": "object"
                 }

log-alert/fetchers/parseable.py

@@ -41,6 +41,10 @@ class ParseableLogFetcher(LogFetcher):
             if labelNum > 0:
                 query += ' AND '
             query += f'log LIKE \'%{filters["text"]}%\''
+        if "timestamp" in filters:
+            if labelNum > 0:
+                query += ' AND '
+            query += f'{filters["timestamp"]} >= {start_time} AND {filters["timestamp"]} < {end_time}'
         payload = {
             "query": query,
             "startTime": datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%dT%H:%M:%SZ"),
@@ -57,9 +61,13 @@ class ParseableLogFetcher(LogFetcher):
                 if timestamp >= start_time and timestamp < end_time:
                     logs.append({
                         "timestamp": item.get("p_timestamp"),
+                        "fetcher-start-time": start_time,
+                        "fetcher-end-time": end_time,
                         "log": item.get("log"),
                         "labels": {k: v for k, v in item.items() if k not in ["p_timestamp", "log"]}
                     })
+                    logger.debug(f"Fetched log: {logs[-1]}")
+            logger.debug(f"Fetched {len(logs)} logs from Parseable between {start_time} and {end_time}")
             return logs
         except requests.exceptions.RequestException as e:
             logger.error(f"Error fetching logs from Parseable: {e}")

log-alert/filters/__init__.py

@@ -1,7 +1,8 @@
 """Filters package for log-alert."""
 
 from .base import Filter
-from .regexp import RegexpFilter
 from .geolocation import GeolocationFilter
+from .regexp import RegexpFilter
+from .timestamp import TimestampFilter
 
-__all__ = ["Filter", "RegexpFilter", "GeolocationFilter"]
+__all__ = ["Filter", "GeolocationFilter", "TimestampFilter", "RegexpFilter"]

log-alert/filters/timestamp.py

@@ -0,0 +1,46 @@
+"""timestamp filter implementation."""
+
+import logging
+import requests
+import datetime
+from typing import Dict, Any, Optional
+from zoneinfo import ZoneInfo
+
+from .base import Filter
+
+logger = logging.getLogger("log-alert")
+
+
+class TimestampFilter(Filter):
+    """Concrete implementation for Timestamp filter."""
+
+    def __init__(self, config: Dict[str, Any]):
+        self.source_field = config["source-field"]
+        self.timezone = config.get("timezone", "UTC")
+        self.timestamp_format = config.get("timestamp-format")
+
+    def filter(self, log: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        timestamp_string = log.get("labels", {}).get(self.source_field)
+        if timestamp_string is None:
+            logger.warning(f"No timestamp found in log for field {self.source_field}")
+            return None
+        else:
+            try:
+                if self.timestamp_format:
+                    logger.debug(f"Parsing timestamp {timestamp_string} with format {self.timestamp_format}")
+                    parsed_time = datetime.datetime.strptime(timestamp_string, self.timestamp_format)
+                    if parsed_time.year == 1900:
+                        # If the year is not specified, assume the current year
+                        parsed_time = parsed_time.replace(year=datetime.datetime.now().year)
+                    timestamp_value = int(parsed_time.replace(tzinfo=ZoneInfo(self.timezone)).timestamp())
+                else:
+                    timestamp_value = int(timestamp_string)
+            except Exception as e:
+                logger.error(f"Error parsing timestamp {timestamp_value} with format {self.timestamp_format}: {e}")
+
+            if timestamp_value >= log.get("fetcher-start-time", 0) and timestamp_value < log.get("fetcher-end-time", float('inf')):
+                logger.debug(f"Log timestamp {timestamp_value} is within the fetcher time range.")
+                return log
+            else:
+                logger.debug(f"Log timestamp {timestamp_value} is outside the fetcher time range.")
+                return None

log-alert/rules/simple.py

@@ -5,7 +5,7 @@ from typing import Dict, Any, Optional
 
 from .base import AlertRule
 from fetchers import LogFetcher
-from filters import RegexpFilter, GeolocationFilter
+from filters import GeolocationFilter, RegexpFilter, TimestampFilter
 from alerters import Alerter
 
 logger = logging.getLogger("log-alert")
@@ -20,10 +20,12 @@ class SimpleAlertRule(AlertRule):
         self.check_interval = config.get("check-interval", 60)
         self.filters = []
         for filter in config.get("filters", []):
-            if filter["type"] == "regexp":
+            if filter["type"] == "geolocation":
-                self.filters.append(RegexpFilter(filter["config"]))
-            elif filter["type"] == "geolocation":
                 self.filters.append(GeolocationFilter(filter["config"]))
+            elif filter["type"] == "regexp":
+                self.filters.append(RegexpFilter(filter["config"]))
+            elif filter["type"] == "timestamp":
+                self.filters.append(TimestampFilter(filter["config"]))
             else:
                 raise ValueError(f"Unsupported filter type: {filter['type']}")
         self.alerter = alerters[config["alerter"]["name"]]