Integrating Symantec Cloud Web Gateway to FortiSIEM

[somasundaram]

Symantec CSWG has a limitation we don't have direct access to integrate cloud secure web gateway to FortiSIEM.

The only option is to download the logs. so I've made a Python script to overcome this limitation.

Workflow of Script :

1) The script will download the logs as a ZIP file in the temp folder

2) Once Downloaded that particular file will contain another .Gz file that will be extracted.

3) After extraction, there will be a text file that contains the logs from Symantec CWSG.

4) After fetching the logs from the txt file it will forward the logs to Supervisor IP in Syslog format.

Python Package dependencies

// requests – To send HTTP requests to download the logs.

// gzip – To handle and decompress .gz files.

// syslog – To send the log data as syslog messages to a remote server.

// shutil – For managing file and directory operations like removing files after processing.

# dnf install pip or dnf install python3-pip

# pip install requests or pip3 install requests

---------------------------------------------------------------------------------------------------------------

Download the Script and place the file in the desired root directory.

# /root/download_and_send_logs.py

# using realpath followed by file name we can find the location of the file.

Once the Script is placed in the desired location, Make the Script as executable file

# chmod +x download_and_send_logs.py

Execute the following command to create an empty log file and set the proper permissions

# touch /var/log/log_downloader.log

# chmod 666 /var/log/log_downloader.log

---------------------------------------------------------------------------------------------------------------

Automate the Script Using Cron

Step 1: Execute the command

#  crontab -e

Step 2: Paste the command to execute Cron

#  */5 * * * * /usr/bin/python3 /root/download_and_send_logs.py >> /var/log/log_downloader.log 2>&1

Save and exit the file.

Step 3: Restart cron services

# systemctl restart crond

import os
import sys
import gzip
import requests
import zipfile
import shutil
import syslog
import socket
from datetime import datetime

# Configuration
SIEM_SERVER = 'SuperIP/FQDN'  # Change to FQDN if needed, e.g., 'siem.example.com'
TEMP_DIR = '/tmp/logs'  # Temporary directory for storing downloaded and extracted logs
SYMPORTAL_URL = "https://portal.threatpulse.com/reportpod/logs/sync"
API_USERNAME = "API_USERNAME"
API_PASSWORD = "API_PASSWORD"

def resolve_syslog_server(syslog_server):
    """Check if syslog_server is an IP or an FQDN and resolve if necessary."""
    try:
        # Check if syslog_server is an IP address
        socket.inet_aton(syslog_server)
        return syslog_server  # Return IP if valid
    except socket.error:
        try:
            # Resolve the FQDN to an IP address
            resolved_ip = socket.gethostbyname(syslog_server)
            print(f"Resolved {syslog_server} to {resolved_ip}")
            return resolved_ip
        except socket.gaierror:
            print(f"Error: Unable to resolve {syslog_server}")
            return None

def download_logs(start_epoch, end_epoch):
    """Download logs from Symantec portal."""
    params = {
        'startDate': start_epoch,
        'endDate': end_epoch,
        'token': 'none'
    }

    headers = {
        'X-APIUsername': API_USERNAME,
        'X-APIPassword': API_PASSWORD
    }

    response = requests.get(SYMPORTAL_URL, params=params, headers=headers, stream=True)

    if response.status_code == 200:
        zip_file_path = os.path.join(TEMP_DIR, 'logs.zip')
        with open(zip_file_path, 'wb') as f:
            shutil.copyfileobj(response.raw, f)
        print(f"Logs downloaded successfully: {zip_file_path}")
        return zip_file_path
    else:
        print(f"Failed to download logs: {response.status_code}")
        return None

def extract_zip(file_path, extract_to):
    """Extract ZIP file into specified directory."""
    with zipfile.ZipFile(file_path, 'r') as zip_ref:
        zip_ref.extractall(extract_to)
    print(f"Extracted ZIP file: {file_path}")

def process_log_file(file_path, syslog_server):
    """Process log file and send each log entry to the syslog server."""
    resolved_ip = resolve_syslog_server(syslog_server)
    if not resolved_ip:
        print("Failed to resolve syslog server. Exiting.")
        return

    # Open the .gz file and read the log lines inside the .txt file
    try:
        if file_path.endswith('.gz'):
            with gzip.open(file_path, 'rt') as f:
                logs = f.readlines()
        else:
            with open(file_path, 'r') as f:
                logs = f.readlines()

        # Send each log entry as a syslog message
        for log in logs:
            syslog.syslog(syslog.LOG_INFO, log.strip())
            print(f"Sent log entry to {resolved_ip}: {log.strip()}")

    except Exception as e:
        print(f"Error processing log file {file_path}: {e}")

def delete_temp_logs():
    """Delete logs from the temp directory after transmission."""
    if os.path.exists(TEMP_DIR):
        shutil.rmtree(TEMP_DIR)
        print(f"Temp directory {TEMP_DIR} and all logs have been deleted.")
    else:
        print(f"Temp directory {TEMP_DIR} does not exist, nothing to delete.")

def get_epoch_time():
    """Get the current epoch time rounded to the nearest 5-minute interval in milliseconds."""
    now = datetime.utcnow()
    minutes = (now.minute // 5) * 5  # Round down to the nearest 5-minute interval
    start_time = now.replace(minute=minutes, second=0, microsecond=0)
    epoch_time = int(start_time.timestamp() * 1000)  # Convert to milliseconds
    return epoch_time

def main():
    # Create the temp directory if it doesn't exist
    if not os.path.exists(TEMP_DIR):
        os.makedirs(TEMP_DIR)

    # Get the epoch time for the current 5-minute interval
    start_epoch = get_epoch_time()
    end_epoch = start_epoch + (5 * 60 * 1000)  # 5 minutes later

    # Download logs
    log_file = download_logs(start_epoch, end_epoch)

    if log_file:
        # Extract ZIP files
        extracted_dir = os.path.join(TEMP_DIR, 'extracted')
        os.makedirs(extracted_dir, exist_ok=True)
        extract_zip(log_file, extracted_dir)

        # Process extracted logs (example: `cloud_40487_20250218050001.log.gz`)
        for extracted_file in os.listdir(extracted_dir):
            extracted_file_path = os.path.join(extracted_dir, extracted_file)
            if extracted_file.endswith('.log.gz'):
                print(f"Processing log file: {extracted_file_path}")
                process_log_file(extracted_file_path, SIEM_SERVER)

        # Clean up and delete temporary logs after processing
        delete_temp_logs()

if __name__ == "__main__":
    main()

Use this command to verify whether the script is running fine.

tail -f /var/log/log_downloader.log

FortiSIEM