spider.py

import sqlite3
import random
import datetime
import os
import joblib
import jieba
import uuid
from datetime import timedelta

# === 配置区域 ===
DB_NAME = 'database.db'
MODEL_PATH = 'models/sentiment_nb.pkl'


def init_db():
    conn = sqlite3.connect(DB_NAME)
    c = conn.cursor()
    # 增加 source 和 sentiment_label 列
    c.execute('''CREATE TABLE IF NOT EXISTS posts
                 (
                     id
                     INTEGER
                     PRIMARY
                     KEY
                     AUTOINCREMENT,
                     content
                     TEXT
                     NOT
                     NULL,
                     source
                     TEXT,
                     sentiment_label
                     INTEGER,
                     publish_time
                     TIMESTAMP
                 )''')
    conn.commit()
    conn.close()


model = None
try:
    if os.path.exists(MODEL_PATH):
        model = joblib.load(MODEL_PATH)
        print("✅ 情感模型加载成功")
except:
    print("⚠️ 使用随机标签")


def predict_sentiment(text):
    if model:
        try:
            words = " ".join(jieba.cut(text))
            return int(model.predict([words])[0])
        except:
            return 0
    return 0


class MockDataGenerator:
    def __init__(self):
        self.conn = sqlite3.connect(DB_NAME)

    def save_data(self, content, source, pub_time):
        cursor = self.conn.cursor()

        # [修改点1] 移除了严格的去重检查，允许相似数据入库
        # 或者我们只检查完全相同且时间也相同的（几乎不可能发生）

        label = predict_sentiment(content)
        try:
            cursor.execute('INSERT INTO posts (content, source, sentiment_label, publish_time) VALUES (?, ?, ?, ?)',
                           (content, source, label, pub_time))
            self.conn.commit()
        except Exception as e:
            print(f"Error: {e}")

    def generate_human_time(self, days_ago=0):
        base_date = datetime.datetime.now() - timedelta(days=days_ago)

        # 权重字典：晚间(19-22点)权重高
        hour_weights = {
            0: 1, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0,
            6: 2, 7: 5, 8: 15,
            9: 20, 10: 25, 11: 30,
            12: 80, 13: 50,
            14: 25, 15: 25, 16: 35, 17: 45,
            18: 70, 19: 100, 20: 120, 21: 110, 22: 90,
            23: 40
        }

        hour = random.choices(list(hour_weights.keys()), weights=list(hour_weights.values()), k=1)[0]
        minute = random.randint(0, 59)
        second = random.randint(0, 59)
        return base_date.replace(hour=hour, minute=minute, second=second, microsecond=0)

    def run(self, total_count=200):
        print(f"\n🧪 正在生成 {total_count} 条数据...")

        # [修改点2] 扩充词库，避免重复
        items = ["电动车", "显示器", "雅思真题", "吉他", "健身卡", "电磁炉", "蓝牙耳机", "宿舍神器", "自行车"]
        locs = ["芙蓉餐厅", "图书馆", "公寓", "三家村", "南光食堂", "海韵公寓", "勤业餐厅"]
        prices = ["50", "100", "200", "500", "800", "一杯奶茶钱"]
        actions = ["出", "求购", "转让", "收"]

        # 更多样化的模板
        templates = [
            "{action}{item}，{loc}自取，价格{price}。",
            "毕业{action}{item}，九成新，{price}带走。",
            "有人出{item}吗？{loc}面交。",
            "【避雷】{loc}今天的饭太难吃了，大家别去。",
            "在{loc}捡到一个{item}，请失主私聊。",
            "吐槽一下，{loc}的网速真的太慢了！",
            "{loc}的风景真的不错，心情变好了。",
            "急{action}{item}，搬家带不走，{price}随便卖。"
        ]

        count = 0
        for days_ago in range(2, -1, -1):
            daily_count = int(total_count / 3)
            for _ in range(daily_count):
                pub_time = self.generate_human_time(days_ago)

                # [修改点3] 随机组合生成内容
                tmpl = random.choice(templates)
                content = tmpl.format(
                    action=random.choice(actions),
                    item=random.choice(items),
                    loc=random.choice(locs),
                    price=random.choice(prices)
                )

                # [修改点4] 添加一个随机后缀以确保唯一性 (可选，这里为了模拟真实感，可以加上编号)
                # content += f" [No.{random.randint(1000,9999)}]"

                # 随机分配来源
                source = random.choice(["校园集市", "表白墙", "微博", "朋友圈"])

                self.save_data(content, source, pub_time)
                count += 1

        print(f"✅ 完成！成功插入 {count} 条数据。不再因为重复被拦截。")

    def close(self):
        self.conn.close()


if __name__ == "__main__":
    init_db()
    num = input("请输入生成数量 (建议 1000): ")
    count = int(num) if num.isdigit() else 1000
    gen = MockDataGenerator()
    gen.run(count)
    gen.close()