""" 大厂AI推荐数据库加载器 用于反扎堆检测功能，加载和查询大厂AI的高频推荐院校。 """ from __future__ import annotations import json import os import re import warnings from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Tuple @dataclass class CrowdRecommendation: """扎堆推荐数据""" name: str # 院校名称 major: str # 专业 frequency: int # 推荐频次（0-4） platforms: List[str] # 推荐平台列表 predicted_increase: int # 预测分数上涨 alternatives: List[Dict[str, Any]] = field(default_factory=list) @property def risk_level(self) -> str: """根据频次计算风险等级（与 crowd_detector._risk_level_from_frequency 一致） frequency == 0: 'none'（不构成扎堆风险） frequency 1: 'low' frequency 2-3: 'medium' frequency >= 4: 'high' """ if self.frequency >= 4: return "high" if self.frequency >= 2: return "medium" if self.frequency >= 1: return "low" return "none" @dataclass class ProvenanceValidation: """T3.2 溯源字段验证结果。 Attributes: ok: 是否通过 schema 校验（errors 为空且非加载失败） errors: schema 硬错误（缺字段、字段类型/取值非法、文件损坏等） warnings: 软警告（如低 confidence、source_url 为空） is_usable: confidence 是否达到 USABLE_CONFIDENCE_THRESHOLD summary: 关键溯源字段的扁平摘要（province/source_type/data_year/...） """ province: str ok: bool errors: List[str] = field(default_factory=list) warnings: List[str] = field(default_factory=list) is_usable: bool = False summary: Dict[str, Any] = field(default_factory=dict) def to_dict(self) -> Dict[str, Any]: return { "province": self.province, "ok": self.ok, "is_usable": self.is_usable, "errors": list(self.errors), "warnings": list(self.warnings), "summary": dict(self.summary), } class CrowdDBLoader: """ 大厂AI推荐数据库加载器 数据存储在 data/crowd_db/{province}.json 文件中。 """ DATA_DIR = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "data", "crowd_db", ) PROVINCE_FILE_MAP = { "北京": "beijing", "天津": "tianjin", "上海": "shanghai", "重庆": "chongqing", "河北": "hebei", "山西": "shanxi", "辽宁": "liaoning", "吉林": "jilin", "黑龙江": "heilongjiang", "江苏": "jiangsu", "浙江": "zhejiang", "安徽": "anhui", "福建": "fujian", "江西": "jiangxi", "山东": "shandong", "河南": "henan", "湖北": "hubei", "湖南": "hunan", "广东": "guangdong", "海南": "hainan", "四川": "sichuan", "贵州": "guizhou", "云南": "yunnan", "陕西": "shaanxi", "甘肃": "gansu", "青海": "qinghai", "新疆": "xinjiang", } # T3.2 溯源查询/验证常量（与 SCHEMA.md 1/3 节保持一致） USABLE_CONFIDENCE_THRESHOLD: float = 0.5 VALID_SOURCE_TYPES: Tuple[str, ...] = ( "manual_summary", "official_release", "platform_scrape", "derived", ) ISO_DATE_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}$") def __init__( self, data_dir: Optional[str] = None, warn_low_confidence: bool = True ): """初始化加载器。 Args: data_dir: 数据目录路径，默认使用 DATA_DIR warn_low_confidence: 低置信度数据是否发出 UserWarning """ self.data_dir = data_dir or self.DATA_DIR self.warn_low_confidence = warn_low_confidence self._cache: Dict[str, dict] = {} def _file_candidates(self, province: str) -> List[str]: slug = self.PROVINCE_FILE_MAP.get(province) candidates: List[str] = [] if slug: candidates.append(f"{slug}.json") candidates.append(f"{province}.json") return candidates def _load_json_file(self, file_path: str) -> Optional[dict]: try: with open(file_path, "r", encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError): return None def load_province(self, province: str) -> Optional[dict]: """加载指定省份的推荐数据。""" if province in self._cache: return self._cache[province] for filename in self._file_candidates(province): file_path = os.path.join(self.data_dir, filename) if not os.path.exists(file_path): continue data = self._load_json_file(file_path) if not data: continue self._cache[province] = data confidence = data.get("confidence") if ( self.warn_low_confidence and isinstance(confidence, (int, float)) and confidence < 0.5 ): warnings.warn( f"{province} 数据置信度较低 ({confidence})，当前仅为骨架数据", UserWarning, stacklevel=2, ) return data return None def load_metadata(self, province: str) -> Optional[dict]: """仅加载省份溯源元数据。""" data = self.load_province(province) if not data: return None return { "province": data.get("province", province), "last_updated": data.get("last_updated", ""), "data_year": data.get("data_year"), "source": data.get("source", ""), "source_url": data.get("source_url", ""), "source_type": data.get("source_type", ""), "confidence": data.get("confidence"), "record_count": sum( len(score_range.get("recommendations", [])) for score_range in data.get("score_ranges", []) if isinstance(score_range, dict) ), } def list_supported_provinces(self) -> List[str]: """返回 loader 支持的省份列表。""" return list(self.PROVINCE_FILE_MAP.keys()) def list_provinces(self) -> List[Dict[str, Any]]: """返回所有支持省份的存在性与元数据概览。""" provinces: List[Dict[str, Any]] = [] for province in self.list_supported_provinces(): file_path = self._resolve_file_path(province) data = ( self._load_json_file(file_path) if file_path and os.path.exists(file_path) else None ) provinces.append( { "province": province, "file_name": os.path.basename(file_path) if file_path else None, "exists": data is not None, "last_updated": data.get("last_updated") if data else None, "data_year": data.get("data_year") if data else None, "source_type": data.get("source_type") if data else None, "confidence": data.get("confidence") if data else None, "record_count": sum( len(score_range.get("recommendations", [])) for score_range in data.get("score_ranges", []) if isinstance(score_range, dict) ) if data else 0, } ) return provinces def _resolve_file_path(self, province: str) -> Optional[str]: for filename in self._file_candidates(province): file_path = os.path.join(self.data_dir, filename) if os.path.exists(file_path): return file_path return None def find_recommendations(self, province: str, score: int) -> List[Dict[str, Any]]: """查询指定分数段内的所有推荐。""" data = self.load_province(province) if not data: return [] results: List[Dict[str, Any]] = [] for score_range in data.get("score_ranges", []): if not isinstance(score_range, dict): continue score_bounds = score_range.get("range") or [] if len(score_bounds) != 2: continue min_score, max_score = score_bounds if min_score <= score <= max_score: results.extend(score_range.get("recommendations", [])) return results def find_recommendation_by_school( self, province: str, school_name: str ) -> Optional[Dict[str, Any]]: """按院校名查询推荐信息（支持模糊匹配）。""" data = self.load_province(province) if not data: return None for score_range in data.get("score_ranges", []): if not isinstance(score_range, dict): continue for rec in score_range.get("recommendations", []): if ( school_name in rec.get("name", "") or rec.get("name", "") in school_name ): return rec return None # ------------------------------------------------------------------ # # T3.2 溯源字段查询 + 验证 # ------------------------------------------------------------------ # REQUIRED_PROVENANCE_FIELDS: Tuple[str, ...] = ( "province", "last_updated", "data_year", "source", "source_type", "confidence", "score_ranges", ) @classmethod def validate_provenance( cls, data: Optional[dict], province: Optional[str] = None ) -> ProvenanceValidation: """T3.2: 校验单省 JSON 顶层溯源字段是否符合 SCHEMA.md。 Args: data: 已加载的省份 dict；为 None 表示加载失败 province: 省份名（用于报告，省略时尝试从 data 取） Returns: ProvenanceValidation 实例，ok 表示 schema 硬错误列表为空 """ prov_name = province or (data.get("province") if data else "") or "" validation = ProvenanceValidation(province=prov_name, ok=False) if data is None: validation.errors.append("load_failed: 省份数据加载失败或文件不存在") return validation # 必填字段 for key in cls.REQUIRED_PROVENANCE_FIELDS: if key not in data: validation.errors.append(f"missing_field: {key}") # province 字符串类型 p = data.get("province") if "province" in data and not isinstance(p, str): validation.errors.append( f"type_error: province 应为 str, got {type(p).__name__}" ) # last_updated ISO 日期（缺失已在 REQUIRED 检查报告） lu = data.get("last_updated") if "last_updated" in data and lu is not None: if not isinstance(lu, str) or not cls.ISO_DATE_PATTERN.match(lu): validation.errors.append( f"format_error: last_updated 应为 YYYY-MM-DD, got {lu!r}" ) elif lu == "": validation.warnings.append("empty_last_updated: last_updated 为空") # data_year 必须为 int（缺失已在 REQUIRED 检查报告） dy = data.get("data_year") if "data_year" in data and dy is not None and not isinstance(dy, int): validation.errors.append( f"type_error: data_year 应为 int, got {type(dy).__name__}" ) # source 非空字符串 src = data.get("source") if "source" in data and (src is None or src == ""): validation.warnings.append("empty_source: source 字段为空") # source_url 可空但须为字符串 su = data.get("source_url") if "source_url" in data and su is not None and not isinstance(su, str): validation.errors.append( f"type_error: source_url 应为 str 或缺失, got {type(su).__name__}" ) # source_type 必须是枚举之一（缺失已在 REQUIRED 检查报告） st = data.get("source_type") if ( "source_type" in data and st is not None and st not in cls.VALID_SOURCE_TYPES ): validation.errors.append( f"enum_error: source_type {st!r} 不在 {list(cls.VALID_SOURCE_TYPES)} 内" ) # confidence 数值 + 区间（缺失已在 REQUIRED 检查报告） c = data.get("confidence") if "confidence" in data and c is not None: if not isinstance(c, (int, float)) or not (0.0 <= c <= 1.0): validation.errors.append( f"range_error: confidence 应在 [0,1], got {c!r}" ) else: if c < cls.USABLE_CONFIDENCE_THRESHOLD: validation.warnings.append( f"low_confidence: confidence={c} < {cls.USABLE_CONFIDENCE_THRESHOLD}" ) validation.is_usable = c >= cls.USABLE_CONFIDENCE_THRESHOLD # score_ranges 必须是 list（缺失已在 REQUIRED 检查报告） sr = data.get("score_ranges") if "score_ranges" in data and sr is not None and not isinstance(sr, list): validation.errors.append( f"type_error: score_ranges 应为 list, got {type(sr).__name__}" ) # source_url 空时记 warning（不影响 ok，但与人工复核流程相关） if isinstance(su, str) and su == "": validation.warnings.append("empty_source_url: source_url 为空") # 摘要（仅在已加载且含核心字段时） if all( k in data for k in ("province", "source_type", "data_year", "confidence") ): validation.summary = { "province": data.get("province"), "source": data.get("source", ""), "source_url": data.get("source_url", ""), "source_type": data.get("source_type"), "data_year": data.get("data_year"), "last_updated": data.get("last_updated", ""), "confidence": data.get("confidence"), } validation.ok = not validation.errors return validation def validate_province(self, province: str) -> ProvenanceValidation: """T3.2: 加载并校验指定省份的溯源字段。 不修改 self._cache 之外的状态；与 warn_low_confidence 兼容。 """ data = self.load_province(province) return self.validate_provenance(data, province=province) def validate_all(self) -> Dict[str, ProvenanceValidation]: """T3.2: 校验所有支持省份的溯源字段。 Returns: {省份名: ProvenanceValidation} """ results: Dict[str, ProvenanceValidation] = {} for province in self.list_supported_provinces(): results[province] = self.validate_province(province) return results def filter_provinces( self, *, source_type: Optional[str] = None, min_confidence: Optional[float] = None, max_confidence: Optional[float] = None, data_year: Optional[int] = None, updated_since: Optional[str] = None, updated_before: Optional[str] = None, only_usable: Optional[bool] = None, ) -> List[str]: """T3.2: 按溯源字段过滤支持省份。 Args: source_type: 仅匹配指定 source_type（如 "manual_summary"） min_confidence: 最低 confidence（含） max_confidence: 最高 confidence（含） data_year: 仅匹配指定 data_year updated_since: YYYY-MM-DD 起始日期（含） updated_before: YYYY-MM-DD 截止日期（含） only_usable: True 仅返回 confidence >= 阈值的省份 Returns: 匹配条件的省份名列表（按 PROVINCE_FILE_MAP 顺序） Note: 仅依据已加载的顶层溯源字段过滤；底层推荐条目不在过滤范围内。 """ results: List[str] = [] for province in self.list_supported_provinces(): data = self.load_province(province) if data is None: continue if source_type is not None and data.get("source_type") != source_type: continue c = data.get("confidence") if min_confidence is not None: if not isinstance(c, (int, float)) or c < min_confidence: continue if max_confidence is not None: if not isinstance(c, (int, float)) or c > max_confidence: continue if data_year is not None and data.get("data_year") != data_year: continue lu = data.get("last_updated") if updated_since is not None: if not isinstance(lu, str) or lu < updated_since: continue if updated_before is not None: if not isinstance(lu, str) or lu > updated_before: continue if only_usable is True: if not ( isinstance(c, (int, float)) and c >= self.USABLE_CONFIDENCE_THRESHOLD ): continue results.append(province) return results def get_provenance_report( self, *, only_usable: bool = False, source_type: Optional[str] = None, ) -> Dict[str, Any]: """T3.2: 汇总各省份溯源字段 + 验证结果。 Args: only_usable: True 时仅包含 confidence >= 阈值的省份 source_type: 同时按 source_type 过滤 Returns: 报告 dict：包含 total/usable_count/failed_count/by_source_type/items """ provinces = self.filter_provinces( source_type=source_type, only_usable=only_usable or None ) items: List[Dict[str, Any]] = [] for province in provinces: validation = self.validate_province(province) item = validation.to_dict() file_path = self._resolve_file_path(province) item["file_name"] = os.path.basename(file_path) if file_path else None items.append(item) by_source_type: Dict[str, int] = {} for item in items: st = (item.get("summary") or {}).get("source_type") or "unknown" by_source_type[st] = by_source_type.get(st, 0) + 1 return { "total": len(items), "usable_count": sum(1 for i in items if i["is_usable"]), "failed_count": sum(1 for i in items if not i["ok"]), "by_source_type": by_source_type, "items": items, } if __name__ == "__main__": loader = CrowdDBLoader() data = loader.load_province("湖南") if data: print(f"✅ 加载湖南数据: {len(data.get('score_ranges', []))} 个分数段") else: print("❌ 加载湖南数据失败") recs = loader.find_recommendations("湖南", score=575) print(f"📊 575分在湖南的扎堆院校: {len(recs)} 个") for rec in recs: print( f" - {rec['name']} {rec['major']} (频次:{rec['frequency']}, +{rec['predicted_increase']}分)" )