【草船借箭】3.数据预处理(Pandas)

# 取得全部上游数据的文件路径
from typing import List
from pathlib import Path
from common import UPSTREAM_DIR

upstream_files: List[Path] = []
for filepath in UPSTREAM_DIR.glob('*.log'):
    upstream_files.append(filepath)

# 初始化一个数组来存储结果
# 解析上游数据
import re
pattern = re.compile(r'^([\d/]+\s[\d:.]+)\s([\d.]+):(\d+)\s?(.*?)\s?(\S*)\s?(\S*)$')

records = []
for upstream_file in upstream_files:
    with open(upstream_file, 'r', encoding='utf-8') as f:
        for line in f.readlines():
            if match := pattern.match(line):
                record = {
                    'timestamp': match.group(1),
                    'ip': match.group(2),
                    'port': match.group(3),
                    'client': match.group(4),
                    'username': match.group(5),
                    'password': match.group(6),
                }
                records.append(record)

# CSV文件输出
import csv
from common import DOWNSTREAM_DIR

fieldnames = ['timestamp', 'ip', 'port', 'client', 'username', 'password']
csv_path = DOWNSTREAM_DIR / "output_3.csv"

with open(csv_path, 'w', newline='', encoding='utf-8') as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
    writer.writeheader()
    for record in records:
        writer.writerow(record)