diff --git a/.gitignore b/.gitignore index 3d9705e1..a52df4d8 100644 --- a/.gitignore +++ b/.gitignore @@ -239,10 +239,7 @@ yarn-error.log* faiss.index rag_data/chunks.json app/core/rag_data/chunks.json -<<<<<<< HEAD -======= summary_outputs/ ->>>>>>> dev *.log old_front/ *.zip diff --git a/.test_filename_generation.py.icloud b/.test_filename_generation.py.icloud deleted file mode 100644 index 6af7132c..00000000 Binary files a/.test_filename_generation.py.icloud and /dev/null differ diff --git a/app/config/__init__.py b/app/config/__init__.py index 644cecb8..e730bf53 100644 --- a/app/config/__init__.py +++ b/app/config/__init__.py @@ -131,8 +131,6 @@ def overlap_sentences(self): """Get overlap sentences from config.""" return self.config['chunking']['overlap_sentences'] -<<<<<<< HEAD -======= # Regulation fetch configuration @property def regulation_fetch_days_back(self): @@ -147,5 +145,4 @@ def summary_output_dir(self): rel_path = self.config['summary']['output_dir'] return str(project_root / rel_path) ->>>>>>> dev config = Config() \ No newline at end of file diff --git a/app/config/development.yml b/app/config/development.yml index da0cf966..4475742d 100644 --- a/app/config/development.yml +++ b/app/config/development.yml @@ -31,8 +31,6 @@ chunking: chunk_words: 500 overlap_sentences: 1 -<<<<<<< HEAD -======= # Regulation fetching configuration regulation_fetch: days_back: 30 # Default days to look back for new regulations @@ -41,7 +39,6 @@ regulation_fetch: summary: output_dir: summary_outputs ->>>>>>> dev # Embedding model configuration embedding: # Default model to use for embeddings diff --git a/app/config/production.yml b/app/config/production.yml index 97898e4d..2ddcc9db 100644 --- a/app/config/production.yml +++ b/app/config/production.yml @@ -19,8 +19,6 @@ build_faiss: docs_data: path: data/ -<<<<<<< HEAD -======= # Chunking configuration chunking: chunk_words: 500 @@ -34,7 +32,6 @@ regulation_fetch: summary: output_dir: summary_outputs ->>>>>>> dev # Embedding model configuration embedding: # Default model to use for embeddings diff --git a/app/config/production.yml.example b/app/config/production.yml.example index 97898e4d..2ddcc9db 100644 --- a/app/config/production.yml.example +++ b/app/config/production.yml.example @@ -19,8 +19,6 @@ build_faiss: docs_data: path: data/ -<<<<<<< HEAD -======= # Chunking configuration chunking: chunk_words: 500 @@ -34,7 +32,6 @@ regulation_fetch: summary: output_dir: summary_outputs ->>>>>>> dev # Embedding model configuration embedding: # Default model to use for embeddings diff --git a/app/core/.example_usage.py.icloud b/app/core/.example_usage.py.icloud deleted file mode 100644 index ca266e00..00000000 Binary files a/app/core/.example_usage.py.icloud and /dev/null differ diff --git a/app/core/auto_update_pipeline.py b/app/core/auto_update_pipeline.py index a5fd12fc..8add59ac 100644 --- a/app/core/auto_update_pipeline.py +++ b/app/core/auto_update_pipeline.py @@ -18,13 +18,9 @@ # Add the app directory to Python path for imports sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from config import config -<<<<<<< HEAD - -======= # Configure logging first logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) ->>>>>>> dev # Import pipeline components from incremental_pipeline import IncrementalPipeline @@ -44,11 +40,6 @@ logger.warning(f"⚠️ Data fetcher not available: {e}") DATA_FETCHER_AVAILABLE = False -<<<<<<< HEAD -# Configure logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) -======= # Import incremental summary component try: from incremental_summary import IncrementalSummary @@ -56,7 +47,6 @@ except ImportError as e: logger.warning(f"⚠️ Incremental summary not available: {e}") SUMMARY_AVAILABLE = False ->>>>>>> dev class AutoUpdatePipeline: """ @@ -85,14 +75,11 @@ def __init__(self, model: str = None, days_back: int = None): # Initialize core pipeline self.pipeline = IncrementalPipeline(model=self.model) -<<<<<<< HEAD -======= # Initialize incremental summary if available self.summary_manager = None if SUMMARY_AVAILABLE: self.summary_manager = IncrementalSummary() ->>>>>>> dev # Paths self.data_dir = Path(config.docs_data_path) @@ -101,10 +88,7 @@ def __init__(self, model: str = None, days_back: int = None): logger.info(f"🗓️ Days back: {self.days_back}") logger.info(f"📁 Data directory: {self.data_dir}") logger.info(f"🌐 Data fetcher available: {DATA_FETCHER_AVAILABLE}") -<<<<<<< HEAD -======= logger.info(f"📄 Incremental summary available: {SUMMARY_AVAILABLE}") ->>>>>>> dev def check_for_new_regulations(self) -> Dict: """ @@ -306,8 +290,6 @@ def run_full_auto_update(self) -> Dict: # Step 3: Run full incremental update (includes new downloads) incremental_result = self.pipeline.full_incremental_update() -<<<<<<< HEAD -======= # Step 4: Run incremental summary update summary_result = None if self.summary_manager: @@ -317,7 +299,6 @@ def run_full_auto_update(self) -> Dict: else: logger.info("📄 Skipping summary update (not available)") ->>>>>>> dev # Calculate totals total_cost = incremental_result['total_cost'] end_time = time.time() @@ -333,10 +314,7 @@ def run_full_auto_update(self) -> Dict: 'regulations_check': check_result, 'download_result': download_result, 'incremental_result': incremental_result, -<<<<<<< HEAD -======= 'summary_result': summary_result, ->>>>>>> dev 'total_cost': total_cost, 'duration_seconds': round(end_time - start_time, 2), 'status': 'success' @@ -364,8 +342,6 @@ def run_incremental_update(self) -> Dict: try: result = self.pipeline.full_incremental_update() -<<<<<<< HEAD -======= # Run incremental summary update summary_result = None if self.summary_manager: @@ -375,20 +351,15 @@ def run_incremental_update(self) -> Dict: else: logger.info("📄 Skipping summary update (not available)") ->>>>>>> dev logger.info("✅ Incremental update completed!") logger.info(f" - Files deleted: {len(result['cleanup_result']['deleted_files'])}") logger.info(f" - Files processed: {len(result['process_result']['processed_files'])}") logger.info(f" - Total cost: ${result['total_cost']:.4f}") -<<<<<<< HEAD - return result -======= return { **result, 'summary_result': summary_result } ->>>>>>> dev except Exception as e: logger.error(f"❌ Incremental update failed: {e}") @@ -448,8 +419,6 @@ def process_specific_files(self, file_paths: List[str]) -> Dict: successful_files = [r['file_path'] for r in results if r['status'] == 'success'] -<<<<<<< HEAD -======= # Generate summaries for processed files summary_result = None if self.summary_manager and successful_files: @@ -457,7 +426,6 @@ def process_specific_files(self, file_paths: List[str]) -> Dict: summary_result = self.summary_manager.generate_summary_for_specific_files(successful_files) logger.info(f"📄 Summary generation completed: {summary_result['status']}") ->>>>>>> dev logger.info(f"✅ Specific file processing completed:") logger.info(f" - Files processed: {len(successful_files)}/{len(resolved_paths)}") logger.info(f" - Total cost: ${total_cost:.4f}") @@ -468,10 +436,7 @@ def process_specific_files(self, file_paths: List[str]) -> Dict: return { 'processed_files': successful_files, 'failed_files': errors, -<<<<<<< HEAD -======= 'summary_result': summary_result, ->>>>>>> dev 'total_cost': total_cost, 'status': 'success' if not errors else 'partial_success' } diff --git a/app/core/compare.py b/app/core/compare.py deleted file mode 100644 index ddf610b6..00000000 --- a/app/core/compare.py +++ /dev/null @@ -1,644 +0,0 @@ -import openai -import json -import re -import numpy as np -import faiss -from typing import List, Dict, Tuple, Optional -import tiktoken -from dotenv import load_dotenv -import os -from datetime import datetime -from collections import defaultdict - -class SectionBySectionRuleComparator: - def __init__(self, faiss_index_path: str, metadata_path: str, api_key: str): - self.api_key = api_key - self.client = openai.OpenAI(api_key=api_key) - - # Load FAISS index and metadata - self.index = faiss.read_index(faiss_index_path) - with open(metadata_path, 'r') as f: - self.metadata = json.load(f) - - # Token management - self.encoding = tiktoken.encoding_for_model("gpt-4") - self.max_tokens_per_section = 15000 # Conservative limit per section comparison - self.max_chunk_tokens = 1000 # Max tokens per individual chunk - - def count_tokens(self, text: str) -> int: - """Count tokens in a text string""" - return len(self.encoding.encode(text)) - - def truncate_text(self, text: str, max_tokens: int) -> str: - """Truncate text to fit within token limit""" - tokens = self.encoding.encode(text) - if len(tokens) <= max_tokens: - return text - return self.encoding.decode(tokens[:max_tokens]) + "..." - - def organize_chunks_by_section(self, chunks: List[Dict]) -> Dict[str, List[Dict]]: - """ - Organize chunks by their section headers - """ - sections = defaultdict(list) - for chunk in chunks: - section_header = chunk.get('section_header', 'General') - # Normalize section headers for better matching - normalized_header = self.normalize_section_header(section_header) - sections[normalized_header].append(chunk) - - # Sort chunks within each section by similarity score - for section in sections: - sections[section].sort(key=lambda x: x.get('similarity_score', 0), reverse=True) - - return dict(sections) - - def normalize_section_header(self, header: str) -> str: - """ - Normalize section headers to improve matching between rules - """ - if not header: - return "General" - - # Convert to lowercase and remove extra whitespace - normalized = re.sub(r'\s+', ' ', header.lower().strip()) - - # Remove common prefixes/suffixes that might differ between years - normalized = re.sub(r'^(section|part|chapter)\s+', '', normalized) - normalized = re.sub(r'\s+(section|part|chapter)$', '', normalized) - - # Remove year references that might make sections appear different - normalized = re.sub(r'\b(20\d{2}|cy\s*20\d{2})\b', '', normalized) - - # Remove common Roman numerals and numbers that might differ - normalized = re.sub(r'\b[ivxlcdm]+\.\s*', '', normalized) - normalized = re.sub(r'^\d+\.\s*', '', normalized) - - return normalized.strip() or "General" - - def find_matching_sections(self, rule1_sections: Dict, rule2_sections: Dict) -> List[Tuple[str, str, float]]: - """ - Find matching sections between two rules using semantic similarity - Returns list of (rule1_section, rule2_section, similarity_score) - """ - matches = [] - - for section1_name, section1_chunks in rule1_sections.items(): - best_match = None - best_score = 0 - - # Create a representative text for this section - section1_text = self.create_section_summary(section1_chunks) - section1_embedding = self.embed_query(section1_text) - - for section2_name, section2_chunks in rule2_sections.items(): - # Skip if already matched - if any(match[1] == section2_name for match in matches): - continue - - section2_text = self.create_section_summary(section2_chunks) - section2_embedding = self.embed_query(section2_text) - - # Calculate semantic similarity - similarity = np.dot(section1_embedding[0], section2_embedding[0]) / ( - np.linalg.norm(section1_embedding[0]) * np.linalg.norm(section2_embedding[0]) - ) - - if similarity > best_score: - best_score = similarity - best_match = section2_name - - if best_match and best_score > 0.3: # Minimum similarity threshold - matches.append((section1_name, best_match, best_score)) - - # Sort by similarity score (highest first) - matches.sort(key=lambda x: x[2], reverse=True) - return matches - - def create_section_summary(self, chunks: List[Dict], max_chunks: int = 5) -> str: - """ - Create a concise summary of a section from its chunks - """ - if not chunks: - return "" - - # Take top chunks by similarity score - top_chunks = chunks[:max_chunks] - - summary_parts = [] - for chunk in top_chunks: - # Truncate long chunks - text = self.truncate_text(chunk['text'], 200) # Short summary - summary_parts.append(text) - - return " ".join(summary_parts) - - def compare_single_section(self, rule1: Dict, rule2: Dict, - rule1_section: str, rule2_section: str, - rule1_chunks: List[Dict], rule2_chunks: List[Dict], - section_similarity: float, topic: str) -> Dict: - """ - Compare a single section between two rules - """ - # Prepare context for this section - rule1_context = self.prepare_section_context(rule1_chunks, rule1_section) - rule2_context = self.prepare_section_context(rule2_chunks, rule2_section) - - # Check token count and truncate if needed - total_tokens = self.count_tokens(rule1_context) + self.count_tokens(rule2_context) - if total_tokens > self.max_tokens_per_section: - # Reduce chunk count proportionally - reduction_factor = self.max_tokens_per_section / total_tokens - rule1_reduced = int(len(rule1_chunks) * reduction_factor * 0.5) - rule2_reduced = int(len(rule2_chunks) * reduction_factor * 0.5) - - rule1_context = self.prepare_section_context(rule1_chunks[:rule1_reduced], rule1_section) - rule2_context = self.prepare_section_context(rule2_chunks[:rule2_reduced], rule2_section) - - prompt = f"""Compare this specific section between two CMS rules: - -**Focus Topic**: {topic} - -**Rule 1**: {rule1['program']} {rule1['year']} {rule1['rule_type']} Rule -**Section**: {rule1_section} -{rule1_context} - -**Rule 2**: {rule2['program']} {rule2['year']} {rule2['rule_type']} Rule -**Section**: {rule2_section} -{rule2_context} - -**Section Similarity Score**: {section_similarity:.2f} - -Please provide a focused comparison of ONLY this section, including: - -1. **Key Changes**: What specifically changed in this section? -2. **Impact**: How do these changes affect healthcare providers? -3. **Details**: Important numbers, dates, or requirements that changed -4. **Significance**: Rate the importance of changes (High/Medium/Low) - -Keep the analysis focused and specific to this section only. Be concise but thorough. - -Section Comparison:""" - - try: - response = self.client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "system", "content": "You are a healthcare policy expert specializing in detailed section-by-section rule analysis."}, - {"role": "user", "content": prompt} - ], - temperature=0.2, - max_tokens=1500 - ) - - return { - "rule1_section": rule1_section, - "rule2_section": rule2_section, - "similarity_score": section_similarity, - "comparison": response.choices[0].message.content, - "chunk_counts": { - "rule1": len(rule1_chunks), - "rule2": len(rule2_chunks) - }, - "token_usage": { - "rule1_context": self.count_tokens(rule1_context), - "rule2_context": self.count_tokens(rule2_context) - } - } - - except Exception as e: - return { - "rule1_section": rule1_section, - "rule2_section": rule2_section, - "similarity_score": section_similarity, - "comparison": f"Error analyzing section: {str(e)}", - "error": True - } - - def prepare_section_context(self, chunks: List[Dict], section_name: str) -> str: - """ - Prepare context string for a specific section - """ - if not chunks: - return f"[{section_name}]: No content available" - - context = f"[{section_name}]\n" - for i, chunk in enumerate(chunks): - # Add chunk with truncation - truncated_text = self.truncate_text(chunk['text'], self.max_chunk_tokens) - context += f"\n{i+1}. {truncated_text}\n" - - return context - - def identify_unmatched_sections(self, rule1_sections: Dict, rule2_sections: Dict, - matches: List[Tuple[str, str, float]]) -> Tuple[List[str], List[str]]: - """ - Identify sections that don't have matches in the other rule - """ - matched_rule1 = {match[0] for match in matches} - matched_rule2 = {match[1] for match in matches} - - unmatched_rule1 = [section for section in rule1_sections.keys() if section not in matched_rule1] - unmatched_rule2 = [section for section in rule2_sections.keys() if section not in matched_rule2] - - return unmatched_rule1, unmatched_rule2 - - def analyze_unmatched_sections(self, rule: Dict, unmatched_sections: List[str], - sections_dict: Dict, rule_label: str) -> Dict: - """ - Analyze sections that exist in only one rule - """ - if not unmatched_sections: - return {"sections": [], "analysis": "No unique sections found."} - - # Limit to top 5 most significant unmatched sections - sections_analysis = [] - for section_name in unmatched_sections[:5]: - chunks = sections_dict[section_name] - summary = self.create_section_summary(chunks, max_chunks=3) - - sections_analysis.append({ - "section_name": section_name, - "chunk_count": len(chunks), - "summary": summary[:500] # Limit summary length - }) - - prompt = f"""Analyze these sections that appear ONLY in {rule_label} ({rule['program']} {rule['year']} {rule['rule_type']}): - -{json.dumps(sections_analysis, indent=2)} - -Provide: -1. **New Content**: What new topics/requirements are introduced? -2. **Significance**: Why were these sections added/removed? -3. **Impact**: How does this affect healthcare providers? - -Analysis:""" - - try: - response = self.client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "system", "content": "You are a healthcare policy expert analyzing unique rule sections."}, - {"role": "user", "content": prompt} - ], - temperature=0.2, - max_tokens=800 - ) - - return { - "sections": sections_analysis, - "analysis": response.choices[0].message.content - } - - except Exception as e: - return { - "sections": sections_analysis, - "analysis": f"Error analyzing unique sections: {str(e)}" - } - - def generate_final_summary(self, section_comparisons: List[Dict], - rule1_unique: Dict, rule2_unique: Dict, - rule1: Dict, rule2: Dict, query: str) -> str: - """ - Generate a comprehensive final summary from all section comparisons - """ - # Categorize sections by significance - high_impact = [] - medium_impact = [] - low_impact = [] - - for comp in section_comparisons: - if not comp.get('error'): - analysis = comp['comparison'].lower() - if 'high' in analysis and 'significance' in analysis: - high_impact.append(comp) - elif 'medium' in analysis and 'significance' in analysis: - medium_impact.append(comp) - else: - low_impact.append(comp) - - prompt = f"""Based on the following section-by-section analysis, provide a comprehensive summary comparing: - -**Query**: {query} -**Rule 1**: {rule1['program']} {rule1['year']} {rule1['rule_type']} Rule -**Rule 2**: {rule2['program']} {rule2['year']} {rule2['rule_type']} Rule - -**HIGH IMPACT SECTIONS** ({len(high_impact)} sections): -{self.summarize_section_group(high_impact)} - -**MEDIUM IMPACT SECTIONS** ({len(medium_impact)} sections): -{self.summarize_section_group(medium_impact)} - -**UNIQUE TO RULE 1**: -{rule1_unique['analysis']} - -**UNIQUE TO RULE 2**: -{rule2_unique['analysis']} - -Provide a comprehensive executive summary including: - -1. **Overview**: What are the main themes of changes? -2. **Critical Changes**: Top 5 most important changes for providers -3. **Financial Impact**: Key financial/payment changes -4. **Implementation Timeline**: When do changes take effect? -5. **Action Items**: What should healthcare providers do? - -Executive Summary:""" - - try: - response = self.client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "system", "content": "You are a senior healthcare policy consultant providing executive summaries."}, - {"role": "user", "content": prompt} - ], - temperature=0.3, - max_tokens=2000 - ) - - return response.choices[0].message.content - - except Exception as e: - return f"Error generating final summary: {str(e)}" - - def summarize_section_group(self, section_group: List[Dict]) -> str: - """ - Create a brief summary of a group of section comparisons - """ - if not section_group: - return "No sections in this category." - - summaries = [] - for comp in section_group[:3]: # Limit to top 3 - section_name = comp['rule1_section'] - # Extract first sentence of comparison for brevity - first_sentence = comp['comparison'].split('.')[0] + "." - summaries.append(f"• {section_name}: {first_sentence}") - - if len(section_group) > 3: - summaries.append(f"• ... and {len(section_group) - 3} more sections") - - return "\n".join(summaries) - - def compare_rules(self, query: str) -> Dict: - """ - Main method for section-by-section rule comparison - """ - print(f"Starting section-by-section comparison for: {query}") - - # Step 1: Parse the query (reuse existing method) - rule1, rule2, topic = self.parse_comparison_query(query) - print(f"Rule 1: {rule1}") - print(f"Rule 2: {rule2}") - print(f"Topic: {topic}") - - # Step 2: Get all chunks for each rule - query_embedding = self.embed_query(query) - rule1_all_chunks = self.filter_chunks_by_rule(rule1) - rule2_all_chunks = self.filter_chunks_by_rule(rule2) - - if not rule1_all_chunks or not rule2_all_chunks: - return {"error": "Could not find chunks for one or both rules"} - - # Step 3: Get relevant chunks (larger set for section analysis) - rule1_relevant = self.semantic_similarity_search(query_embedding, rule1_all_chunks, k=150) - rule2_relevant = self.semantic_similarity_search(query_embedding, rule2_all_chunks, k=150) - - #print(f"Found {len(rule1_relevant)} relevant chunks in rule 1") - #print(f"Found {len(rule2_relevant)} relevant chunks in rule 2") - - # Step 4: Organize chunks by section - rule1_sections = self.organize_chunks_by_section(rule1_relevant) - rule2_sections = self.organize_chunks_by_section(rule2_relevant) - - #print(f"Rule 1 sections: {list(rule1_sections.keys())}") - #print(f"Rule 2 sections: {list(rule2_sections.keys())}") - - # Step 5: Find matching sections - section_matches = self.find_matching_sections(rule1_sections, rule2_sections) - #print(f"Found {len(section_matches)} matching section pairs") - - # Step 6: Compare each matching section - section_comparisons = [] - for rule1_section, rule2_section, similarity in section_matches: - #print(f"Comparing: {rule1_section} <-> {rule2_section} (similarity: {similarity:.2f})") - - comparison = self.compare_single_section( - rule1, rule2, rule1_section, rule2_section, - rule1_sections[rule1_section], rule2_sections[rule2_section], - similarity, topic - ) - section_comparisons.append(comparison) - - # Step 7: Analyze unmatched sections - unmatched_rule1, unmatched_rule2 = self.identify_unmatched_sections( - rule1_sections, rule2_sections, section_matches - ) - - rule1_unique = self.analyze_unmatched_sections( - rule1, unmatched_rule1, rule1_sections, "Rule 1" - ) - rule2_unique = self.analyze_unmatched_sections( - rule2, unmatched_rule2, rule2_sections, "Rule 2" - ) - - # Step 8: Generate final comprehensive summary - final_summary = self.generate_final_summary( - section_comparisons, rule1_unique, rule2_unique, rule1, rule2, query - ) - - return { - "rule1": rule1, - "rule2": rule2, - "topic": topic, - "section_comparisons": section_comparisons, - "rule1_unique_sections": rule1_unique, - "rule2_unique_sections": rule2_unique, - "final_summary": final_summary, - "stats": { - "total_sections_compared": len(section_comparisons), - "rule1_unique_sections": len(unmatched_rule1), - "rule2_unique_sections": len(unmatched_rule2), - "rule1_total_chunks": len(rule1_relevant), - "rule2_total_chunks": len(rule2_relevant) - } - } - - # Include existing methods from original class - def parse_comparison_query(self, query: str) -> Tuple[Dict, Dict, str]: - """Parse user query to extract rules and topic (from original class)""" - query_lower = query.lower() - - # Extract programs - programs = [] - if "mpfs" in query_lower: - programs.append("MPFS") - if "snf" in query_lower: - programs.append("SNF") - if "hospice" in query_lower: - programs.append("Hospice") - - # Extract years - years = re.findall(r'\b(20\d{2})\b', query) - years = [int(year) for year in years] - - # Extract rule types - rule_types = [] - if "final" in query_lower: - rule_types.append("Final") - if "proposed" in query_lower: - rule_types.append("Proposed") - - # Extract topic/focus - topic = self._extract_topic(query_lower) - - # Create rule dictionaries - rule1, rule2 = self._create_rule_dicts(programs, years, rule_types, topic) - - return rule1, rule2, topic - - def _extract_topic(self, query_lower: str) -> str: - """Extract the main topic from the query""" - topic_keywords = { - "fee schedule": "fee schedule", - "payment": "payment", - "wage index": "wage index", - "quality": "quality", - "reporting": "reporting", - "cap amount": "cap amount", - "update": "update" - } - - for keyword, topic in topic_keywords.items(): - if keyword in query_lower: - return topic - - return "general comparison" - - def _create_rule_dicts(self, programs: List[str], years: List[int], - rule_types: List[str], topic: str) -> Tuple[Dict, Dict]: - """Create two rule dictionaries from extracted information""" - program = programs[0] if programs else "General" - - if len(years) == 2 and len(rule_types) == 1: - rule1 = {"program": program, "year": years[0], "rule_type": rule_types[0], "topic": topic} - rule2 = {"program": program, "year": years[1], "rule_type": rule_types[0], "topic": topic} - elif len(years) == 1 and len(rule_types) == 2: - rule1 = {"program": program, "year": years[0], "rule_type": rule_types[0], "topic": topic} - rule2 = {"program": program, "year": years[0], "rule_type": rule_types[1], "topic": topic} - elif len(years) == 2 and len(rule_types) == 2: - rule1 = {"program": program, "year": years[0], "rule_type": rule_types[0], "topic": topic} - rule2 = {"program": program, "year": years[1], "rule_type": rule_types[1], "topic": topic} - else: - current_date_time = datetime.now() - rule1 = {"program": program, "year": years[0] if years else current_date_time.year - 1, "rule_type": "Final", "topic": topic} - rule2 = {"program": program, "year": years[-1] if len(years) > 1 else current_date_time.year, "rule_type": "Final", "topic": topic} - - return rule1, rule2 - - def embed_query(self, query: str) -> np.ndarray: - """Create embedding for the user query""" - response = self.client.embeddings.create( - model="text-embedding-3-small", - input=query - ) - return np.array([response.data[0].embedding], dtype="float32") - - def filter_chunks_by_rule(self, rule: Dict) -> List[Dict]: - """Filter metadata chunks by rule dictionary""" - filtered_chunks = [] - - for chunk in self.metadata: - meta = chunk.get("metadata", {}) - - # Check program match - if rule["program"].lower() not in meta.get("program", "").lower(): - continue - - # Check year match - if meta.get("year") != rule["year"]: - continue - - # Check rule type match - if rule["rule_type"].lower() not in meta.get("rule_type", "").lower(): - continue - - filtered_chunks.append(chunk) - - return filtered_chunks - - def semantic_similarity_search(self, query_embedding: np.ndarray, - rule_chunks: List[Dict], k: int = 50) -> List[Dict]: - """Find most relevant chunks from a specific rule using semantic similarity""" - if not rule_chunks: - return [] - - # Search directly in the main FAISS index - distances, indices = self.index.search(query_embedding, k * 2) - - # Create a set of rule chunk texts for fast lookup - rule_chunk_texts = {chunk["text"] for chunk in rule_chunks} - - # Filter results to only include chunks from the specified rule - relevant_chunks = [] - for dist, idx in zip(distances[0], indices[0]): - if idx < len(self.metadata): - chunk_data = self.metadata[idx] - if chunk_data["text"] in rule_chunk_texts: - chunk = chunk_data.copy() - chunk['similarity_score'] = 1 / (1 + dist) - relevant_chunks.append(chunk) - - if len(relevant_chunks) >= k: - break - - return relevant_chunks - - -# Example usage -if __name__ == "__main__": - load_dotenv() - - comparator = SectionBySectionRuleComparator( - faiss_index_path="./rag_data/faiss.index", - metadata_path="./rag_data/faiss_metadata.json", - api_key=os.getenv("OPENAI_API_KEY") - ) - - # Test query - query = "Compare MPFS fee schedule for 2023 final rule and 2024 final rule" - - print(f"Testing section-by-section comparison: {query}") - result = comparator.compare_rules(query) - print (result) - - # if "error" in result: - # print(f"Error: {result['error']}") - # else: - # # print(f"\n{'='*80}") - # # print("SECTION-BY-SECTION COMPARISON RESULTS") - # # print('='*80) - - # # print(f"\nRules Compared:") - # # print(f"Rule 1: {result['rule1']['program']} {result['rule1']['year']} {result['rule1']['rule_type']}") - # # print(f"Rule 2: {result['rule2']['program']} {result['rule2']['year']} {result['rule2']['rule_type']}") - - # # #print(f"\nStats:") - # # stats = result['stats'] - # # print(f"- Sections compared: {stats['total_sections_compared']}") - # # print(f"- Rule 1 unique sections: {stats['rule1_unique_sections']}") - # # print(f"- Rule 2 unique sections: {stats['rule2_unique_sections']}") - - # print(f"\n{'='*60}") - # print("FINAL EXECUTIVE SUMMARY") - # print('='*60) - # print(result['final_summary']) - - # print(f"\n{'='*60}") - # print("DETAILED SECTION COMPARISONS") - # print('='*60) - # for i, comp in enumerate(result['section_comparisons'][:10], 1): # Show first 5 - # if not comp.get('error'): - # print(f"\n{i}. {comp['rule1_section']} <-> {comp['rule2_section']}") - # #print(f" Similarity: {comp['similarity_score']:.2f}") - # print(f" {comp['comparison']}") \ No newline at end of file diff --git a/app/core/preprocessing/config_loader.py b/app/core/preprocessing/config_loader.py index 2f605776..c3780b98 100644 --- a/app/core/preprocessing/config_loader.py +++ b/app/core/preprocessing/config_loader.py @@ -2,11 +2,7 @@ Configuration Loader Provides unified configuration loading and management for the preprocessing package, -<<<<<<< HEAD -integrating with the main application config system. -======= integrating with the main application config system and .env file. ->>>>>>> dev Example: # Load configuration @@ -31,8 +27,6 @@ from typing import Dict, Any, Optional import logging -<<<<<<< HEAD -======= # Load .env file try: from dotenv import load_dotenv @@ -57,7 +51,6 @@ except ImportError: print("⚠️ python-dotenv not available, .env file not loaded") ->>>>>>> dev logger = logging.getLogger(__name__) @@ -97,18 +90,11 @@ def _load_app_config(self) -> None: from config import config self._app_config = config -<<<<<<< HEAD - logger.debug("Loaded main application config") - - except ImportError as e: - logger.warning(f"Could not load main application config: {e}") -======= logger.debug("✅ Loaded main application config") except ImportError as e: logger.warning(f"❌ Could not load main application config: {e}") logger.warning(" Falling back to standalone configuration") ->>>>>>> dev self._app_config = None def _get_from_app_config(self, attribute: str, default: Any = None) -> Any: @@ -155,10 +141,7 @@ def get_embedding_config(self) -> Dict[str, Any]: config = loader.get_embedding_config() builder = FAISSBuilder(**config) """ -<<<<<<< HEAD -======= # Start with defaults ->>>>>>> dev config = { 'model': 'text-embedding-3-small', 'api_key': os.getenv('OPENAI_API_KEY'), @@ -167,11 +150,6 @@ def get_embedding_config(self) -> Dict[str, Any]: 'rate_limit_delay': 1.0 } -<<<<<<< HEAD - # Override from app config - if self._app_config: - config['model'] = self._get_from_app_config('default_embedding_model', config['model']) -======= # Override from app config if available if self._app_config: config['model'] = self._get_from_app_config('default_embedding_model', config['model']) @@ -187,7 +165,6 @@ def get_embedding_config(self) -> Dict[str, Any]: }) except Exception as e: logger.warning(f"Could not load model config from app: {e}") ->>>>>>> dev # Override from manual config config.update(self._config_override.get('embedding', {})) @@ -220,17 +197,6 @@ def get_paths_config(self) -> Dict[str, Path]: 'tracking_file': project_root / 'rag_data' / 'file_tracking.json' } -<<<<<<< HEAD - # Override from app config - if self._app_config: - try: - config['data_directory'] = Path(self._get_from_app_config('docs_data_path', str(config['data_directory']))) - config['output_directory'] = Path(self._get_from_app_config('build_faiss_output_folder', str(config['output_directory']))) - config['faiss_index'] = Path(self._get_from_app_config('faiss_index_path', str(config['faiss_index']))) - config['faiss_metadata'] = Path(self._get_from_app_config('faiss_metadata_path', str(config['faiss_metadata']))) - except Exception as e: - logger.warning(f"Error loading paths from app config: {e}") -======= # Override from app config if available if self._app_config: try: @@ -246,7 +212,6 @@ def get_paths_config(self) -> Dict[str, Path]: except Exception as e: logger.warning(f"❌ Error loading paths from app config: {e}") logger.warning(" Using default paths") ->>>>>>> dev # Override from manual config paths_override = self._config_override.get('paths', {}) @@ -270,10 +235,7 @@ def get_processing_config(self) -> Dict[str, Any]: paths = self.get_paths_config() chunking = self.get_chunking_config() embedding = self.get_embedding_config() -<<<<<<< HEAD -======= regulation_fetch = self.get_regulation_fetch_config() ->>>>>>> dev config = { 'data_dir': paths['data_directory'], @@ -282,11 +244,7 @@ def get_processing_config(self) -> Dict[str, Any]: 'model': embedding['model'], 'chunk_words': chunking['chunk_words'], 'overlap_sentences': chunking['overlap_sentences'], -<<<<<<< HEAD - 'days_back': 30 # Default for regulation fetching -======= 'days_back': regulation_fetch['days_back'] ->>>>>>> dev } # Override from manual config @@ -294,8 +252,6 @@ def get_processing_config(self) -> Dict[str, Any]: return config -<<<<<<< HEAD -======= def get_regulation_fetch_config(self) -> Dict[str, Any]: """ Get regulation fetching configuration. @@ -324,7 +280,6 @@ def get_regulation_fetch_config(self) -> Dict[str, Any]: return config ->>>>>>> dev def get_validation_config(self) -> Dict[str, Any]: """ Get system validation configuration. diff --git a/app/core/preprocessing/faiss_builder.py b/app/core/preprocessing/faiss_builder.py index 4f3a962c..4b9e8a1d 100644 --- a/app/core/preprocessing/faiss_builder.py +++ b/app/core/preprocessing/faiss_builder.py @@ -51,12 +51,8 @@ def __init__( model: str = "text-embedding-3-small", batch_size: int = 50, max_retries: int = 5, -<<<<<<< HEAD - rate_limit_delay: float = 1.0 -======= rate_limit_delay: float = 1.0, model_config: Optional[Dict[str, Any]] = None ->>>>>>> dev ): """ Initialize FAISSBuilder with OpenAI configuration. @@ -67,10 +63,7 @@ def __init__( batch_size: Number of texts to process per API call max_retries: Maximum retry attempts for failed API calls rate_limit_delay: Base delay between API calls in seconds -<<<<<<< HEAD -======= model_config: Optional model configuration dict (price, encoding, etc.) ->>>>>>> dev Example: builder = FAISSBuilder( @@ -90,34 +83,6 @@ def __init__( self.max_retries = max_retries self.rate_limit_delay = rate_limit_delay -<<<<<<< HEAD - # Model configuration and pricing - self.model_configs = { - "text-embedding-3-small": { - "price_per_1k_tokens": 0.00002, - "max_tokens": 8191, - "encoding": "cl100k_base", - "dimension": 1536 - }, - "text-embedding-ada-002": { - "price_per_1k_tokens": 0.0001, - "max_tokens": 8191, - "encoding": "text-embedding-ada-002", - "dimension": 1536 - }, - "text-embedding-3-large": { - "price_per_1k_tokens": 0.00013, - "max_tokens": 8191, - "encoding": "cl100k_base", - "dimension": 3072 - } - } - - if model not in self.model_configs: - raise ProcessingError(f"Unsupported model: {model}. Supported models: {list(self.model_configs.keys())}") - - self.config = self.model_configs[model] -======= # Store provided model config self._model_config = model_config @@ -151,7 +116,6 @@ def __init__( raise ProcessingError(f"Unsupported model: {model}. Supported models: {list(self.model_configs.keys())}") self.config = self.model_configs[model] ->>>>>>> dev # Set up tokenizer if self.config["encoding"] == "cl100k_base": diff --git a/app/core/preprocessing/incremental_manager.py b/app/core/preprocessing/incremental_manager.py index a3e319c8..470e86a0 100644 --- a/app/core/preprocessing/incremental_manager.py +++ b/app/core/preprocessing/incremental_manager.py @@ -29,11 +29,7 @@ import os from pathlib import Path -<<<<<<< HEAD -from typing import Dict, List, Optional, Union, Any, Set -======= from typing import Dict, List, Optional, Union, Any, Set, Tuple ->>>>>>> dev import logging from datetime import datetime @@ -98,11 +94,6 @@ def __init__( overlap_sentences=overlap_sentences ) -<<<<<<< HEAD - self.faiss_builder = FAISSBuilder( - api_key=api_key, - model=model -======= # Create FAISS builder with full configuration from .config_loader import ConfigLoader config_loader = ConfigLoader() @@ -125,7 +116,6 @@ def __init__( max_retries=embedding_config.get('max_retries', 5), rate_limit_delay=embedding_config.get('rate_limit_delay', 1.0), model_config=model_config ->>>>>>> dev ) self.file_tracker = FileTracker( @@ -195,8 +185,6 @@ def _remove_file_chunks(self, chunks: List[Dict[str, Any]], filename: str) -> Tu return filtered_chunks, removed_count -<<<<<<< HEAD -======= def _rebuild_index_by_reorganization(self, original_chunks: List[Dict[str, Any]], remaining_chunks: List[Dict[str, Any]], chunks_removed: int) -> Dict[str, Any]: """ Efficiently remove vectors from FAISS index using remove_ids (following incremental_faiss.py pattern). @@ -388,7 +376,6 @@ def _fallback_to_full_rebuild(self, remaining_chunks: List[Dict[str, Any]]) -> D else: raise ProcessingError(f"Fallback rebuild failed: {build_result.get('error')}") ->>>>>>> dev @handle_operation("file processing", success_fields={'chunks_added': 0, 'cost': 0.0}) def process_file(self, file_path: Union[str, Path]) -> Dict[str, Any]: """ @@ -622,29 +609,11 @@ def remove_file(self, file_path: Union[str, Path]) -> Dict[str, Any]: save_result = self._save_chunks(filtered_chunks) ensure_success(save_result, "chunk file update") -<<<<<<< HEAD - # Step 4: Rebuild FAISS index without this file's data -======= # Step 4: Rebuild FAISS index efficiently without regenerating embeddings ->>>>>>> dev embeddings_removed = 0 rebuild_cost = 0.0 if filtered_chunks: -<<<<<<< HEAD - # Rebuild index from remaining chunks - logger.info(f"Rebuilding FAISS index without {filename}") - - build_result = self.faiss_builder.build_index_from_chunks(filtered_chunks, "flat") - ensure_success(build_result, "index rebuild") - - embeddings_removed = chunks_removed # Approximate - rebuild_cost = build_result['total_cost'] - - # Save rebuilt index - save_index_result = self.faiss_builder.save_index(self.index_path, self.metadata_path) - ensure_success(save_index_result, "rebuilt index save") -======= # Use efficient index reorganization instead of full rebuild logger.info(f"Reorganizing FAISS index without {filename} (no embedding regeneration)") @@ -657,7 +626,6 @@ def remove_file(self, file_path: Union[str, Path]) -> Dict[str, Any]: # Save reorganized index save_index_result = self.faiss_builder.save_index(self.index_path, self.metadata_path) ensure_success(save_index_result, "reorganized index save") ->>>>>>> dev else: # No chunks left, remove index files if self.index_path.exists(): diff --git a/app/core/preprocessing/pipeline.py b/app/core/preprocessing/pipeline.py index 5c545c20..40d79afc 100644 --- a/app/core/preprocessing/pipeline.py +++ b/app/core/preprocessing/pipeline.py @@ -718,28 +718,6 @@ def schedule_auto_update( ) try: -<<<<<<< HEAD - # Choose pipeline type - if args.auto_update or args.check_regulations or args.download: - pipeline = AutoUpdatePipeline( - data_dir=args.data_dir, - output_dir=args.output_dir, - api_key=args.api_key, - model=args.model, - chunk_words=args.chunk_words, - overlap_sentences=args.overlap, - days_back=args.days_back - ) - else: - pipeline = ProcessingPipeline( - data_dir=args.data_dir, - output_dir=args.output_dir, - api_key=args.api_key, - model=args.model, - chunk_words=args.chunk_words, - overlap_sentences=args.overlap - ) -======= # Load base configuration from config files from .config_loader import ConfigLoader config_loader = ConfigLoader() @@ -772,7 +750,6 @@ def schedule_auto_update( pipeline = AutoUpdatePipeline(**final_config) else: pipeline = ProcessingPipeline(**final_config) ->>>>>>> dev # Execute requested operation if args.status: diff --git a/app/core/preprocessing/utils/__init__.py b/app/core/preprocessing/utils/__init__.py index a2f4b00d..66330523 100644 --- a/app/core/preprocessing/utils/__init__.py +++ b/app/core/preprocessing/utils/__init__.py @@ -10,23 +10,15 @@ from .file_tracker import FileTracker from .data_persistence import DataPersistence -<<<<<<< HEAD -from .error_handler import handle_operation, ProcessingError -======= from .error_handler import handle_operation, ProcessingError, ensure_success, combine_results ->>>>>>> dev from .system_validator import SystemValidator __all__ = [ 'FileTracker', 'DataPersistence', 'handle_operation', -<<<<<<< HEAD - 'ProcessingError', -======= 'ProcessingError', 'ensure_success', 'combine_results', ->>>>>>> dev 'SystemValidator' ] \ No newline at end of file diff --git a/app/core/search.py b/app/core/search.py index 0ed8b292..a163625a 100644 --- a/app/core/search.py +++ b/app/core/search.py @@ -21,9 +21,6 @@ 3.4) Compute Confidence 4) Summary (TODO) """ -<<<<<<< HEAD -import os -======= import sys import os from pathlib import Path @@ -34,7 +31,6 @@ if str(project_root) not in sys.path: sys.path.insert(0, str(project_root)) ->>>>>>> dev os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" import numpy as np import faiss @@ -42,11 +38,7 @@ import json from typing import List, Dict, Any import logging -<<<<<<< HEAD -# from key import OPENAI_API_KEY -======= from rank_bm25 import BM25Okapi ->>>>>>> dev OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") from sklearn.feature_extraction.text import TfidfVectorizer @@ -56,11 +48,6 @@ class ChatSearchService: -<<<<<<< HEAD - def __init__(self, openai_api_key: str, faiss_index_path: str = "./rag_data/faiss.index", - metadata_path: str = "./rag_data/faiss_metadata.json"): - self.openai_client = openai.OpenAI(api_key=openai_api_key) -======= def __init__(self, openai_api_key: str, faiss_index_path: str = None, metadata_path: str = None): """ @@ -88,7 +75,6 @@ def __init__(self, openai_api_key: str, faiss_index_path: str = None, faiss_index_path = faiss_index_path or "./rag_data/faiss.index" metadata_path = metadata_path or "./rag_data/faiss_metadata.json" ->>>>>>> dev self.faiss_index = faiss.read_index(faiss_index_path) with open(metadata_path, 'r', encoding='utf-8') as f: @@ -105,12 +91,6 @@ def __init__(self, openai_api_key: str, faiss_index_path: str = None, self.doc_texts = [chunk['text'] for chunk in self.all_chunks] self.sparse_matrix = self.tfidf_vectorizer.fit_transform(self.doc_texts) -<<<<<<< HEAD - def embed_text(self, text: str) -> np.ndarray: - response = self.openai_client.embeddings.create( - #model="text-embedding-ada-002", - model = "text-embedding-3-small", -======= self.tokenized_docs = [doc.split() for doc in self.doc_texts] self.bm25 = BM25Okapi(self.tokenized_docs) @@ -133,70 +113,11 @@ def embed_text(self, text: str) -> np.ndarray: response = self.openai_client.embeddings.create( model=model, ->>>>>>> dev input=text ) embedding = np.array(response.data[0].embedding, dtype='float32') return embedding / np.linalg.norm(embedding) -<<<<<<< HEAD - # def search(self, query: str, filters: Dict[str, Any] = None, top_k: int = 20) -> List[Dict]: - # # Step 1: Apply filters if present - # if filters: - # filtered_chunks = [] - # for i, chunk in enumerate(self.all_chunks): - # if all(chunk.get("metadata", {}).get(k) == v for k, v in filters.items()): - # chunk['__original_index__'] = i - # filtered_chunks.append(chunk) - # - # if not filtered_chunks: - # return [] - # - # embeddings = [self.faiss_index.reconstruct(chunk['__original_index__']) for chunk in filtered_chunks] - # doc_matrix = np.vstack(embeddings).astype("float32") - # doc_matrix /= np.linalg.norm(doc_matrix, axis=1, keepdims=True) - # else: - # doc_matrix = np.vstack([self.faiss_index.reconstruct(i) for i in range(self.faiss_index.ntotal)]) - # doc_matrix /= np.linalg.norm(doc_matrix, axis=1, keepdims=True) - # filtered_chunks = self.all_chunks.copy() - # - # # Step 2: Embed and normalize the query - # query_embedding = self.embed_text(query).reshape(1, -1) - # - # # Step 3: Compute cosine similarity - # similarities = np.dot(doc_matrix, query_embedding.T).squeeze() - # - # # Step 4: Hybrid: Get BM25 (TF-IDF) similarity scores - # sparse_query_vec = self.tfidf_vectorizer.transform([query]) - # sparse_similarities = cosine_similarity(sparse_query_vec, self.sparse_matrix).flatten() - # - # # Step 5: Combine scores (weighted sum) - # alpha = 0.3 # weight for embedding, 0.5 for sparse - # combined_scores = alpha * similarities + (1 - alpha) * sparse_similarities[:len(filtered_chunks)] - # - # top_indices = np.argsort(-combined_scores)[:top_k] - # - # # Step 6: Prepare results - # results = [] - # for i in top_indices: - # chunk = filtered_chunks[i].copy() - # chunk["distance"] = float(1 - combined_scores[i]) - # results.append(chunk) - # - # return results - - def search(self, query: str, filters: Dict[str, Any] = None, top_k: int = 20): - query_embedding = self.embed_text(query).reshape(1, -1) - distances, indices = self.faiss_index.search(query_embedding, top_k) - results = [] - for idx, dist in zip(indices[0], distances[0]): - if 0 <= idx < len(self.all_chunks): - chunk = self.all_chunks[idx].copy() - chunk["distance"] = float(dist) - results.append(chunk) - return results - -======= def search(self, query: str, filters: Dict[str, Any] = None, top_k: int = 20): """ Search for relevant chunks with optional filtering. @@ -255,70 +176,40 @@ def search(self, query: str, filters: Dict[str, Any] = None, top_k: int = 20): return results ->>>>>>> dev def generate_answer(self, query: str, chunks: List[Dict], max_context_length: int = 4000) -> Dict[str, Any]: if not chunks: return { "answer": "Sorry, I couldn't find relevant information to answer your question.", "confidence": 0.0, "sources_used": [], -<<<<<<< HEAD -======= "source_file":[], ->>>>>>> dev "total_sources": 0 } context_parts = [] current_length = 0 sources_used = [] -<<<<<<< HEAD -======= source_file=[] ->>>>>>> dev for i, chunk in enumerate(chunks): chunk_text = f"[Source {i+1}] {chunk['text']}" context_parts.append(chunk_text) current_length += len(chunk_text) -<<<<<<< HEAD -======= file_name = chunk.get("metadata", {}).get("source_file", "") source_file.append(file_name) ->>>>>>> dev sources_used.append({ "source_id": i+1, "text_preview": chunk['text'][:100] + "..." if len(chunk['text']) > 100 else chunk['text'], "distance": chunk.get('distance', 0), -<<<<<<< HEAD - "metadata": chunk.get('metadata', {}) -======= "metadata": chunk.get('metadata', {}), "source_file":file_name ->>>>>>> dev }) context = "\n\n".join(context_parts) -<<<<<<< HEAD - prompt = f"""Based on the following medical regulation document content, please answer the user's question. - -Please follow these rules: -1. Only answer based on the provided content, do not add external knowledge -3. Cite relevant sources in your answer using the format [Source1], [Source2], etc. -4. Keep answers accurate, professional, and easy to understand -5. If there are multiple relevant pieces of information, organize them into a clear structure - -Context content: -{context} - -User question: {query} - -Answer:""" -======= prompt = f""" You are a senior expert in medical policy and regulation analysis. Based on the following medical regulation document content, please answer the user's question. Please follow these rules: @@ -335,7 +226,6 @@ def generate_answer(self, query: str, chunks: List[Dict], max_context_length: in User question: {query} Answer:""" ->>>>>>> dev try: response = self.openai_client.chat.completions.create( @@ -358,10 +248,7 @@ def generate_answer(self, query: str, chunks: List[Dict], max_context_length: in "answer": answer, "confidence": round(confidence, 2), "sources_used": sources_used, -<<<<<<< HEAD -======= "source_file":source_file, ->>>>>>> dev "total_sources": len(chunks), "context_length": current_length } @@ -393,8 +280,6 @@ def ask_question(self, query: str, filters: Dict[str, Any] = None, top_k: int = chunks = self.search(query, filters=filters, top_k=top_k) result = self.generate_answer(query, chunks) -<<<<<<< HEAD -======= # Background processing: Extract cited sources and print chunk information self._process_cited_sources_and_print(result, chunks) @@ -405,36 +290,11 @@ def ask_question(self, query: str, filters: Dict[str, Any] = None, top_k: int = # Remove [Source X] citations from answer (sources are handled separately by frontend) result['answer'] = self._remove_citations(result['answer']) ->>>>>>> dev result.update({ "query": query, "filters_applied": filters, "retrieval_method": "filtered" if filters else "unfiltered" }) -<<<<<<< HEAD - return result, chunks - -def ask_query(query): - # Example usage - try: - # Initialize service with actual FAISS index and metadata files - service = ChatSearchService( - openai_api_key=OPENAI_API_KEY, # Ensure you have set your OpenAI API key - faiss_index_path="./rag_data/faiss.index", - metadata_path="./rag_data/faiss_metadata.json" - ) - - result, chunks = service.ask_question(query, top_k=20) - print(f"Question: {result['query']}") - print(f"Answer: {result['answer']}") - #print(f"Confidence: {result['confidence']}") - print(f"Number of sources used: {len(result['sources_used'])}") - if len(result['sources_used']) != 0: - print("\nSource details:") - for source in result['sources_used']: - print(f" - Source {source['source_id']}: {source['text_preview']}") - print(f" Similarity: {1-source['distance']:.3f}") -======= return result, cited_chunks @@ -577,7 +437,6 @@ def ask_query(query): ) result, chunks = service.ask_question(query, top_k=20) ->>>>>>> dev # prepare final output final_output = result['answer'] @@ -586,11 +445,6 @@ def ask_query(query): except Exception as e: print(f"Error: {e}") -<<<<<<< HEAD - print("Please ensure faiss.index and faiss_metadata.json files exist in the ./rag_data/ directory") - print("Also ensure you have set the correct OpenAI API key") - -======= print("Please ensure faiss.index and faiss_metadata.json files exist in the configured directory") print("Also ensure you have set the correct OpenAI API key") @@ -627,4 +481,3 @@ def ask_query(query): print("❌ Error:", e) print("=" * 60 + "\n") ->>>>>>> dev diff --git a/app/core/summarizer.py b/app/core/summarizer.py index 4e806b3d..5585ad80 100644 --- a/app/core/summarizer.py +++ b/app/core/summarizer.py @@ -1,12 +1,6 @@ import os import json import sys -<<<<<<< HEAD -from pathlib import Path -from typing import List, Dict, Any -from openai import OpenAI -from config import config -======= import hashlib import asyncio from pathlib import Path @@ -29,7 +23,6 @@ parent_dir = Path(__file__).parent.parent sys.path.insert(0, str(parent_dir)) from config import config ->>>>>>> dev class SummaryGenerator: @@ -46,16 +39,6 @@ class SummaryGenerator: generator.generate_report(chunk_data, file_name) """ -<<<<<<< HEAD - def __init__(self, output_dir: str = "./summary_outputs", openai_api_key: str = os.getenv("OPENAI_API_KEY")): - if not openai_api_key: - sys.exit("OPENAI_API_KEY is not set.") - self.client = OpenAI(api_key=openai_api_key) - self.summary_dir = Path(output_dir) - self.summary_dir.mkdir(exist_ok=True) - - def _count_tokens(self, text: str) -> int: -======= def __init__(self, output_dir: str = None, openai_api_key: str = os.getenv("OPENAI_API_KEY"), use_async: bool = True): if not openai_api_key: raise ValueError("OPENAI_API_KEY is not set.") @@ -82,13 +65,10 @@ def _count_tokens(self, text: str, model: str = "gpt-4o-mini") -> int: print(f"⚠️ Tiktoken failed, using estimation: {e}") # Fallback estimation (rough approximation) ->>>>>>> dev return len(text.encode("utf-8")) // 4 def _chunk_batches(self, data: List[Dict], batch_size: int) -> List[List[Dict]]: return [data[i:i + batch_size] for i in range(0, len(data), batch_size)] -<<<<<<< HEAD -======= def _get_batch_hash(self, batch: List[Dict]) -> str: """Generate hash for batch content to use as cache key.""" @@ -243,29 +223,12 @@ async def process_batch_with_semaphore(batch_idx, batch): individual_summaries.extend(result) return individual_summaries ->>>>>>> dev def _get_batch_prompt(self, program: str, batch: List[Dict], batch_num: int) -> str: formatted = "\n\n".join( f"[Section {i + 1}]\n{c.get('page_content') or c.get('text', '')}" for i, c in enumerate(batch) ) return f""" -<<<<<<< HEAD -You are a senior compliance analyst reviewing CMS {program.upper()} Final Rule content. - -Below are multiple sections grouped together. For each distinct topic you identify in the text, extract: -- topic (brief title) -- key_changes (bulleted list with conditions, expiration dates, etc.) -- quantitative_data (numbers, percentages, codes, dates) -- stakeholders_affected (e.g. physicians, billing staff) - -Respond as a list of valid JSON objects. - -[Start of batched text: Batch {batch_num}] -{formatted} -[End of text] -""" -======= You are a senior compliance analyst reviewing CMS {program.upper()} Final Rule content. Below are multiple sections grouped together. For each distinct topic you identify in the text, extract: @@ -280,7 +243,6 @@ def _get_batch_prompt(self, program: str, batch: List[Dict], batch_num: int) -> {formatted} [End of text] """ ->>>>>>> dev def _get_final_report_prompt(self, program: str, year: str, summaries: str) -> str: try: @@ -292,22 +254,6 @@ def _get_final_report_prompt(self, program: str, year: str, summaries: str) -> s title = f"Business Intelligence Report: CY {year} {program.upper()} Final Rule" return f""" -<<<<<<< HEAD -You are a senior regulatory analyst. Using the structured JSON below, write a professional summary for executives. - -### {title} - -**Sections to include:** -{section_str} -- Action Items for Stakeholders - -Use specific numbers, dates, and codes. Emphasize what is temporary vs. permanent. - -[Start of structured JSON data] -{summaries} -[End of structured JSON data] -""" -======= You are a senior regulatory analyst. Using the structured JSON below, write a professional summary for executives. ### {title} @@ -322,7 +268,6 @@ def _get_final_report_prompt(self, program: str, year: str, summaries: str) -> s {summaries} [End of structured JSON data] """ ->>>>>>> dev def generate_report(self, chunks_data: List[Dict], file_name: str) -> str: program = "MPFS" @@ -339,9 +284,6 @@ def generate_report(self, chunks_data: List[Dict], file_name: str) -> str: print("📄 Cached summary found. Loading...\n") return summary_path.read_text() -<<<<<<< HEAD - if json_path.exists(): -======= # Check for batch cache first batch_cache = self._load_batch_cache(file_name) if batch_cache.get("completed", False): @@ -355,36 +297,12 @@ def generate_report(self, chunks_data: List[Dict], file_name: str) -> str: individual_summaries.extend(batch_result) print(f"📄 Loaded {len(individual_summaries)} summaries from cache") elif json_path.exists(): ->>>>>>> dev print("📄 Found precomputed JSON summary.") with open(json_path, 'r') as jf: individual_summaries = json.load(jf) else: individual_summaries = [] batches = self._chunk_batches(chunks_data, batch_size=5) -<<<<<<< HEAD - - for b_idx, batch in enumerate(batches): - prompt = self._get_batch_prompt(program, batch, b_idx + 1) - print(f"🔄 Summarizing batch {b_idx + 1}/{len(batches)}...") - - try: - response = self.client.chat.completions.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": prompt}], - temperature=0.0, - response_format={"type": "json_object"} - ) - parsed = json.loads(response.choices[0].message.content) - if isinstance(parsed, list): - individual_summaries.extend(parsed) - else: - individual_summaries.append(parsed) - print(f"✅ Batch {b_idx + 1} summarized.") - except Exception as e: - print(f"❌ Error summarizing batch {b_idx + 1}: {e}") - continue -======= print(f"📄 Processing {len(batches)} batches with caching...") if self.use_async and self.async_client: @@ -407,36 +325,21 @@ def generate_report(self, chunks_data: List[Dict], file_name: str) -> str: batch_result = self._process_batch_sync(program, batch, b_idx, file_name, batch_hash) if batch_result: individual_summaries.extend(batch_result) ->>>>>>> dev if not individual_summaries: return "No report generated; all batch analysis failed." -<<<<<<< HEAD -======= # Mark as completed and save combined JSON batch_cache["completed"] = True index_path = self._get_batch_index_path(file_name) with open(index_path, 'w') as f: json.dump(batch_cache, f, indent=2) ->>>>>>> dev with open(json_path, 'w') as jf: json.dump(individual_summaries, jf, indent=2) print(f"💾 Intermediate JSON saved to {json_path}") year_str = file_name.split('_')[0] if '_' in file_name else "latest" -<<<<<<< HEAD - joined_summaries = json.dumps(individual_summaries, indent=2) - - if self._count_tokens(joined_summaries) > 100000: - print(f"⚠️ Truncating to 100,000 tokens...") - joined_summaries = joined_summaries[:350000] - - final_prompt = self._get_final_report_prompt(program, year_str, joined_summaries) - - try: -======= # Check token count and handle large summaries token_count = self._count_tokens(json.dumps(individual_summaries, indent=2)) @@ -460,23 +363,12 @@ def _generate_single_final_report(self, program: str, year: str, summaries: List joined_summaries = json.dumps(summaries, indent=2) final_prompt = self._get_final_report_prompt(program, year, joined_summaries) ->>>>>>> dev print("\n🔄 Generating final summary report...") final_response = self.client.chat.completions.create( model="gpt-4o-mini", messages=[{"role": "user", "content": final_prompt}], temperature=0.1, ) -<<<<<<< HEAD - final_report = final_response.choices[0].message.content.strip() - with open(summary_path, 'w') as f: - f.write(final_report) - print("✅ Final report saved.") - return final_report - except Exception as e: - print(f"❌ Error generating report: {e}") - return f"Final synthesis failed. Raw data below:\n\n{joined_summaries}" -======= return final_response.choices[0].message.content.strip() except Exception as e: print(f"❌ Error generating single final report: {e}") @@ -635,4 +527,3 @@ def _generate_segmented_final_report(self, program: str, year: str, summaries: L except Exception as e: print(f"❌ Error generating segmented final report: {e}") return f"Segmented final synthesis failed. Raw data below:\n\n{json.dumps(summaries, indent=2)}" ->>>>>>> dev diff --git a/app/main.py b/app/main.py index f0ba4743..9cd3c400 100644 --- a/app/main.py +++ b/app/main.py @@ -6,20 +6,13 @@ import sys import os import logging -<<<<<<< HEAD -from typing import Dict, Any, Optional -======= import json from typing import Dict, Any, Optional, List ->>>>>>> dev from flask import Flask, request, jsonify from flask_cors import CORS from werkzeug.exceptions import BadRequest, HTTPException import yaml -<<<<<<< HEAD -======= import requests ->>>>>>> dev from app.core import summarizer from app.core.summarizer import SummaryGenerator @@ -63,10 +56,6 @@ def create_app() -> Flask: ) summarizer = SummaryGenerator( -<<<<<<< HEAD - output_dir="./summary_outputs", -======= ->>>>>>> dev openai_api_key=api_key ) @@ -79,8 +68,6 @@ def create_app() -> Flask: return app -<<<<<<< HEAD -======= def list_available_documents() -> List[Dict[str, Any]]: """ List all available documents from the data directory. @@ -398,7 +385,6 @@ def list_available_summaries() -> List[Dict[str, Any]]: return summaries ->>>>>>> dev def register_error_handlers(app: Flask) -> None: """ Register error handlers for the Flask application. @@ -457,8 +443,6 @@ def validate_json_request(required_fields: Optional[list[str]] = None) -> Dict[s return data -<<<<<<< HEAD -======= @app.route("/api/documents", methods=["GET"]) def list_documents() -> tuple[Dict[str, Any], int]: """ @@ -476,7 +460,6 @@ def list_documents() -> tuple[Dict[str, Any], int]: logger.error(f"Error in documents endpoint: {str(e)}") return jsonify({"error": str(e)}), 400 ->>>>>>> dev @app.route("/api/chat", methods=["POST"]) def chat() -> tuple[Dict[str, Any], int]: """ @@ -484,31 +467,19 @@ def chat() -> tuple[Dict[str, Any], int]: Request body: { -<<<<<<< HEAD - "query": str # The user's question -======= "query": str, # The user's question "doc_names": List[str] # Optional: List of document names to filter by ->>>>>>> dev } Returns: { -<<<<<<< HEAD - "response": str # The system's response -======= "response": str, # The system's response "sources": List[Dict[str, Any]] # Source information with chunks ->>>>>>> dev } """ try: data = validate_json_request(required_fields=["query"]) query = data.get("query") -<<<<<<< HEAD - result, chunks = chat_service.ask_question(query) - return jsonify({"response": result["answer"]}) -======= doc_names = data.get("doc_names", []) # Optional document filter # Create filters if doc_names are provided @@ -551,7 +522,6 @@ def chat() -> tuple[Dict[str, Any], int]: "response": result["answer"], "sources": sources_list }) ->>>>>>> dev except Exception as e: logger.error(f"Error in chat endpoint: {str(e)}") return jsonify({"error": str(e)}), 400 @@ -580,8 +550,6 @@ def summarize() -> tuple[Dict[str, str], int]: return jsonify({"error": str(e)}), 400 -<<<<<<< HEAD -======= @app.route("/api/get-summary", methods=["POST"]) def api_get_summary() -> tuple[Dict[str, Any], int]: @@ -675,7 +643,6 @@ def get_federal_register_info(doc_number: str) -> tuple[Dict[str, Any], int]: ->>>>>>> dev def main() -> None: """Main entry point for the Flask application.""" app = create_app() diff --git a/cost_verification.py b/cost_verification.py deleted file mode 100644 index a85b500e..00000000 --- a/cost_verification.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 - -print("=== 成本验证 ===") -print("测试数据:") -print("- 总tokens: 14,109,286") -print("- 报告成本: $1.4109") -print() - -print("计算验证:") -tokens = 14109286 -cost = tokens / 1000 * 0.0001 -print(f"- 计算成本: ${cost:.4f}") -print(f"- 差异: ${abs(cost - 1.4109):.6f}") -print(f"- 是否匹配: {abs(cost - 1.4109) < 0.0001}") -print() - -print("详细计算:") -print(f"公式: {tokens} / 1000 * 0.0001") -print(f"步骤1: {tokens} / 1000 = {tokens / 1000}") -print(f"步骤2: {tokens / 1000} * 0.0001 = {cost}") -print() - -print("OpenAI定价验证:") -print("- 模型: text-embedding-ada-002") -print("- 价格: $0.0001 per 1K tokens") -print("- 我们的公式: total_tokens / 1000 * 0.0001") -print("- 公式正确性: ✅ 完全正确") -print() - -print("结论: 成本计算完全正确！") -print("- 我们的 $1.4109 成本与 OpenAI 官方定价完全相符") -print("- 计算公式: total_tokens / 1000 * 0.0001") -print("- 定价模型: text-embedding-ada-002") \ No newline at end of file diff --git a/data/MPFS/.2025_MPFS_proposed_2024-14828.xml.icloud b/data/MPFS/.2025_MPFS_proposed_2024-14828.xml.icloud deleted file mode 100644 index 5599890f..00000000 Binary files a/data/MPFS/.2025_MPFS_proposed_2024-14828.xml.icloud and /dev/null differ diff --git a/docs/RegHealth_PRD(Removed).md b/docs/RegHealth_PRD(Removed).md deleted file mode 100644 index b86d88a4..00000000 --- a/docs/RegHealth_PRD(Removed).md +++ /dev/null @@ -1,165 +0,0 @@ -# RegHealth MVP – Product Requirements Document -**Author:** [Your Name] -**Last Updated:** 27 May 2025 -**Status:** Draft - ---- - -## 1. Purpose & Vision -Professionals who track regulatory or standards‑driven content (e.g., health‑tech, pharma, clinical research) waste hours opening dense XML documents, skimming for relevant changes, and re‑summarising them for colleagues. **RegHealth** turns any XML bundle into an *actionable knowledge notebook*: an always‑visible chat space paired with living summaries, mind‑maps, and change‑tracking—everything cached locally for speed, while the canonical XML lives on the server. - ---- - -## 2. Technical Workflow Update (2024-06) - -### Large XML Handling & Section-Based Processing -- **Partitioning:** Large XML files (often >300MB, >1000 pages) are automatically partitioned into logical sections (e.g., "Medicare Physician Fee Schedule", "HIPAA regulations"). -- **Chunking:** Each section is further split into manageable text chunks for embedding and retrieval. -- **Embedding & Storage:** Chunks are embedded (e.g., via OpenAI API) and stored in a vector database, with metadata for section and location. -- **Section-Level Operations:** All LLM-based features (Q&A, summarization, comparison) operate at the section level to avoid context overflow and ensure performance. -- **API & Frontend:** Backend API and frontend are designed to support section selection and section-level operations. Users can select a section for Q&A, summary, or comparison. -- **Rationale:** - - Enables handling of very large documents without exceeding LLM context limits. - - Improves performance and scalability by isolating operations to relevant sections. - - Lays the foundation for future cross-section or multi-section features. - -### Core Backend Modules (MVP) -- `core/xml_partition.py`: Partition XML into logical sections. -- `core/xml_chunker.py`: Chunk a section into smaller text units. -- `core/embedding.py`: Generate and store embeddings for section chunks. -- `core/llm.py`: Section-level LLM operations (summarization, Q&A, comparison). - -### Extensibility -- The architecture supports future expansion to multi-section or cross-document operations, as well as more advanced caching and retrieval strategies. - ---- - -## 2. Goals & Success Metrics - -| Goal | Measure of Success | Target (MVP) | -| ---- | ----------------- | ------------- | -| Faster comprehension of large XML | Avg. time from upload / selection to first summary ready | ≤ 90 s on 200 MB (≈1,000 pages) file | -| Trusted answers | % of answers containing ≥1 inline citation | ≥ 95 % | -| Continuous use | Median chat turns per session | ≥ 8 | -| Re‑use of cached artefacts | Cache hit rate on repeat file selections | ≥ 80 % | -| High unit‑test coverage | `core/` modules | ≥ 90 % | - ---- - -## 3. Personas & Core Jobs - -| Persona | Job‑to‑be‑Done | Pain Today | Desired Outcome | -| ------- | -------------- | ---------- | --------------- | -| **Regulatory Analyst Alice** | "Spot what changed between two guideline releases." | Manual red‑line diffing, copy/paste into Word. | One‑click trend/comparison view with citations. | -| **Medical Writer Ben** | "Extract sections relevant to device safety." | Endless scrolling, missed updates. | Ask in chat → get paragraph plus source id. | -| **Quality Lead Carla** | "Produce weekly summaries for execs." | Writing recap emails from scratch. | Export mind‑map + bullet summary in seconds. | - ---- - -## 4. Scope (MVP) - -### 4.1 Functional Requirements -1. **Server‑side XML repository** - * Primary flow: user searches and selects XML already in the central library. - * Secondary (lower‑priority) flow: user uploads a private XML file (≤ 200 MB); stored in object storage, processed with same pipeline. -2. **High‑volume ingestion** - * Stream‑based parser and chunker must handle **≥ 200 MB** XML without loading the whole file into RAM. - * Pre‑processing runs as asynchronous server job; UI shows progress and auto‑refreshes when ready. -3. **Interactive mind‑map** - * Auto‑expand top nodes; click node → display source paragraph. -4. **Summaries with source‑aware notes** - * Bullet list per section, each bullet links back (e.g. `§1.2`). -5. **Contextual Q&A chat** - * Multi‑turn; answers show inline citations; memory resets on demand. -6. **Document comparison** - * Select ≥ 2 files → ask aspect (e.g., "dosage changes") → receive diff summary. -7. **History & caching** - * Derived artefacts (summaries, embeddings, mind‑maps) cached in Redis + disk; keyed by server file ID. -8. **NotebookLM‑style three‑column UI** - * **Left:** files & history. **Center:** persistent chat. **Right:** actions/results. - -### 4.2 Non‑Functional Requirements - -| Category | Requirement | -| -------- | ----------- | -| Performance | **Server ingest:** ≤ 90 s to parse, chunk, embed and cache a 200 MB / 1,000‑page XML on a 4‑core node; subsequent queries ≤ 3 s. | -| Storage | XML files stored in central S3‑compatible bucket with versioning; derived artefacts cached locally. | -| Usability | All primary actions reachable in ≤ 2 clicks; keyboard shortcut to focus chat (⌘/Ctrl + K). | -| Reliability | Graceful error messages; autosave chat state every 5 s. | -| Security | Role‑based ACL on library XML; private uploads default to owner‑only; redact XML attributes labelled "PII". | -| Internationalisation | UTF‑8 throughout; token estimates accurate for CJK characters. | -| Accessibility | WCAG 2.1 AA colours; ARIA labels on mind‑map nodes. | - -### 4.3 Out of Scope (MVP) -* Real‑time multi‑user collaboration -* Non‑XML formats (PDF, HTML) -* Fully offline LLM (will rely on OpenAI gpt‑4o via LangChain wrapper) - ---- - -## 5. User Journey (Happy Path) - -1. **Select / Upload File(s)** → XML added to processing queue. -2. **Generate Summary** (Right sidebar) → bullet list & mind‑map appear; cached for future sessions. -3. **Ask Follow‑up** ("What are new safety requirements?") → grounded answer with `[§3.2]` references. -4. **Compare** previous vs. new XML → diff summary; user scrolls chat while results stay in Right column. -5. **Return next week**; cached artefacts load instantly. - ---- - -## 6. Detailed Requirements Traceability - -| Epic / Feature | Key Modules | Acceptance Criteria | -| -------------- | ----------- | ------------------- | -| **XML Parser** | `xml_parser.py` | Streams 200 MB file with < 500 MB peak RAM; passes test suite. | -| **Mind‑map Renderer** | `mindmap_builder.py`, Streamlit component | Markdown → Markmap renders; special chars escaped. | -| **Vector Index & Retrieval** | `index_builder.py`, `retriever.py` | Top‑3 chunks contain ≥1 query keyword in 90 % of tests. | -| **Summariser** | `summarizer.py` | Summary ≤ 30 % original tokens; ≥ 80 % ROUGE‑1 vs. human baseline. | -| **Chat Engine** | `chat_engine.py` | Memory remembers entities across 5 turns; `reset` clears state. | -| **Caching & History** | Redis / disk | Cached artefacts keyed by file ID; load ≤ 2 s. | -| **UI Shell** | `app/main.py` | 3‑column layout stable ≥ 1280 px; responsive down to 1024 px. | - ---- - -## 7. Milestones & Timeline (8 Weeks) - -| Week | Deliverable | Owner | Exit Criteria | -| ---- | ----------- | ----- | ------------- | -| 1 | Repo scaffold, Streamlit hello world | FE Lead | App runs; placeholder 3‑column layout. | -| 2 | XML parser + tests | Backend Lead | CI passes; coverage > 90 %. | -| 3 | Mind‑map component | FE Lead | Renders static markdown demo file. | -| 4 | FAISS index & retrieval | Backend Lead | `get_relevant_chunks` returns sensible top‑3. | -| 5 | Q&A pipeline | AI Lead | Chat answers with citations on sample XML. | -| 6 | Summariser UI | AI & FE | Summary + mind‑map linked; click bullet reveals source. | -| 7 | Stateful chat & comparison | AI Lead | 5‑turn chat demo; diff summary across two docs. | -| 8 | Final polish, Docker, E2E tests | All | One‑command Docker up; 5 scripted E2E tests pass. | - ---- - -## 8. Risks & Mitigations - -| Risk | Likelihood | Impact | Mitigation | -| ---- | ---------- | ------ | ---------- | -| OpenAI rate limits slow summarisation | Medium | High | Batch embeddings; exponential back‑off; optional local model toggle. | -| 200 MB uploads saturate embedding quota | Medium | High | Chunk in 1,000‑token windows; queue jobs with concurrency limits. | -| Long ingest time frustrates users | Medium | Medium | Background job + progress bar; e‑mail/UI notification on completion. | -| Citation drift from LLM hallucinations | Medium | High | Grounded retrieval prompt; highlight low‑similarity answers for review. | -| Mind‑map JS fails in older browsers | Low | Medium | Fallback to outline view. | - ---- - -## 9. Open Questions - -1. What is the maximum expected XML size beyond 200 MB, and do we need sharding? -2. Do private uploads count against a per‑organisation storage quota? -3. Which regulatory domains (FDA, EMA, etc.) will seed demo content? - ---- - -## 10. Appendix - -* **Glossary:** - * **RAG** – Retrieval‑Augmented Generation - * **FAISS** – Facebook AI Similarity Search -* **Relevant Docs:** Architectural Decision Records ADR‑001 – ADR‑005 (link forthcoming) - diff --git a/docs/cloudflare_workers_guide.md b/docs/cloudflare_workers_guide.md deleted file mode 100644 index 0b56a403..00000000 --- a/docs/cloudflare_workers_guide.md +++ /dev/null @@ -1,371 +0,0 @@ -# Cloudflare Workers 完整指南 - -## 🎯 什么是 Cloudflare Workers？ - -Cloudflare Workers 是一个**边缘计算平台**，可以： -- 🌍 在全球 200+ 个数据中心运行你的代码 -- 🔒 自动处理 SSL、CDN、DDoS 防护 -- 💰 完全免费（每天 100,000 个请求） -- 🚀 无需管理服务器 -- ⚡ 超低延迟（通常 < 10ms） - -## 🚀 快速开始 - -### 1. 安装 Wrangler CLI -```bash -# 安装 Wrangler -npm install -g wrangler - -# 登录到 Cloudflare -wrangler login -``` - -### 2. 部署 Workers -```bash -# 进入 workers 目录 -cd workers - -# 部署 -./deploy.sh -``` - -### 3. 测试 API -```bash -# 测试简单聊天 -curl https://reghealth-navigator-api.your-subdomain.workers.dev/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' - -# 测试主聊天 -curl https://reghealth-navigator-api.your-subdomain.workers.dev/api/chat \ - -H "Content-Type: application/json" \ - -d '{"query": "What is Medicare?"}' -``` - -## 🔧 配置前端 - -### GitHub Pages -```bash -# 进入仓库 Settings → Secrets and variables → Actions -# 添加: VITE_API_BASE_URL = https://reghealth-navigator-api.your-subdomain.workers.dev -``` - -### Cloudflare Pages -```bash -# 在 Cloudflare Pages 仪表板中 -# 环境变量: VITE_API_BASE_URL = https://reghealth-navigator-api.your-subdomain.workers.dev -``` - -## 📁 项目结构 - -``` -workers/ -├── worker.js # 主要的 Worker 代码 -├── wrangler.toml # 配置文件 -└── deploy.sh # 部署脚本 -``` - -## 🔍 Workers 代码详解 - -### 主要功能 -1. **CORS 处理** - 允许前端跨域访问 -2. **路由处理** - 根据路径分发请求 -3. **JSON 解析** - 处理请求和响应 -4. **错误处理** - 统一的错误响应 - -### API 端点 -- `GET /` - 健康检查 -- `POST /api/simple-chat` - 简单测试 -- `POST /api/chat` - 主聊天功能 - -## 🛠️ 高级功能 - -### 1. 环境变量 -```toml -# wrangler.toml -[vars] -ENVIRONMENT = "production" -API_VERSION = "1.0.0" -``` - -### 2. 密钥管理 -```bash -# 设置 OpenAI API 密钥 -wrangler secret put OPENAI_API_KEY - -# 在代码中使用 -const apiKey = env.OPENAI_API_KEY; -``` - -### 3. KV 存储 -```toml -# wrangler.toml -[[kv_namespaces]] -binding = "REGHEALTH_DATA" -id = "your-kv-namespace-id" -``` - -```javascript -// 在代码中使用 -await env.REGHEALTH_DATA.put("key", "value"); -const value = await env.REGHEALTH_DATA.get("key"); -``` - -### 4. R2 存储 -```toml -# wrangler.toml -[[r2_buckets]] -binding = "REGHEALTH_FILES" -bucket_name = "reghealth-files" -``` - -```javascript -// 存储文件 -await env.REGHEALTH_FILES.put("file.txt", "content"); - -// 读取文件 -const file = await env.REGHEALTH_FILES.get("file.txt"); -``` - -## 🔄 集成 OpenAI API - -### 完整版本示例 -```javascript -// 在 worker.js 中添加 -async function callOpenAI(query, env) { - const response = await fetch('https://api.openai.com/v1/chat/completions', { - method: 'POST', - headers: { - 'Authorization': `Bearer ${env.OPENAI_API_KEY}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - model: 'gpt-3.5-turbo', - messages: [ - { - role: 'system', - content: 'You are a helpful assistant for Medicare regulations.' - }, - { - role: 'user', - content: query - } - ], - max_tokens: 1000 - }) - }); - - const data = await response.json(); - return data.choices[0].message.content; -} - -// 更新 handleChat 函数 -async function handleChat(request, env) { - try { - const data = await request.json(); - - if (!data.query) { - return new Response(JSON.stringify({ error: 'Missing query field' }), { - status: 400, - headers: { ...corsHeaders, 'Content-Type': 'application/json' }, - }); - } - - // 调用 OpenAI API - const response = await callOpenAI(data.query, env); - - return new Response(JSON.stringify({ response }), { - status: 200, - headers: { ...corsHeaders, 'Content-Type': 'application/json' }, - }); - } catch (error) { - return new Response(JSON.stringify({ error: error.message }), { - status: 500, - headers: { ...corsHeaders, 'Content-Type': 'application/json' }, - }); - } -} -``` - -## 📊 监控和分析 - -### 1. Cloudflare 仪表板 -- 访问 [Cloudflare Dashboard](https://dash.cloudflare.com) -- 查看 Workers 分析 -- 监控请求数量、错误率、响应时间 - -### 2. 日志 -```javascript -// 在代码中添加日志 -console.log('Request received:', request.url); -console.log('Processing query:', data.query); -``` - -### 3. 错误追踪 -```javascript -// 捕获和记录错误 -try { - // 你的代码 -} catch (error) { - console.error('Error:', error); - // 返回错误响应 -} -``` - -## 🔒 安全考虑 - -### 1. API 密钥保护 -```bash -# 使用 wrangler secret 存储敏感信息 -wrangler secret put OPENAI_API_KEY -wrangler secret put DATABASE_URL -``` - -### 2. 请求验证 -```javascript -// 验证请求来源 -const origin = request.headers.get('Origin'); -const allowedOrigins = ['https://yourdomain.com', 'https://*.pages.dev']; - -if (!allowedOrigins.some(allowed => origin?.includes(allowed))) { - return new Response('Unauthorized', { status: 403 }); -} -``` - -### 3. 速率限制 -```javascript -// 简单的速率限制 -const clientIP = request.headers.get('CF-Connecting-IP'); -const rateLimitKey = `rate_limit:${clientIP}`; - -// 检查速率限制 -const requests = await env.KV.get(rateLimitKey) || 0; -if (requests > 100) { // 每分钟100次请求 - return new Response('Rate limit exceeded', { status: 429 }); -} - -// 更新计数器 -await env.KV.put(rateLimitKey, requests + 1, { expirationTtl: 60 }); -``` - -## 💰 成本分析 - -### 免费层限制 -- **请求数**: 每天 100,000 个请求 -- **CPU 时间**: 每天 10,000,000 CPU 毫秒 -- **内存**: 128MB 内存 -- **脚本大小**: 1MB - -### 付费层 -- **Workers Paid**: $5/月 -- **额外请求**: $0.50/百万请求 -- **额外 CPU**: $12.50/百万 CPU 毫秒 - -## 🚨 限制和注意事项 - -### 1. 执行时间限制 -- 免费版: 10 秒 -- 付费版: 30 秒 - -### 2. 内存限制 -- 免费版: 128MB -- 付费版: 1GB - -### 3. 网络请求 -- 支持 fetch API -- 可以调用外部 API -- 有网络超时限制 - -### 4. 文件系统 -- 无本地文件系统 -- 使用 KV 或 R2 存储数据 - -## 🔄 从 Flask 迁移 - -### 主要差异 -| Flask | Cloudflare Workers | -|-------|-------------------| -| Python | JavaScript | -| 本地文件系统 | KV/R2 存储 | -| 长时间运行 | 无状态 | -| 传统服务器 | 边缘计算 | - -### 迁移步骤 -1. **转换路由逻辑** -2. **替换文件操作** -3. **调整数据库连接** -4. **测试和部署** - -## 🎯 最佳实践 - -### 1. 代码组织 -```javascript -// 分离关注点 -const handlers = { - '/api/chat': handleChat, - '/api/simple-chat': handleSimpleChat, -}; - -// 主处理函数 -async function handleRequest(request, env) { - const url = new URL(request.url); - const handler = handlers[url.pathname]; - - if (handler) { - return handler(request, env); - } - - return new Response('Not found', { status: 404 }); -} -``` - -### 2. 错误处理 -```javascript -// 统一的错误处理 -function createErrorResponse(message, status = 500) { - return new Response(JSON.stringify({ error: message }), { - status, - headers: { ...corsHeaders, 'Content-Type': 'application/json' }, - }); -} -``` - -### 3. 性能优化 -```javascript -// 缓存响应 -const cacheKey = `cache:${request.url}`; -const cached = await env.KV.get(cacheKey); - -if (cached) { - return new Response(cached, { - headers: { ...corsHeaders, 'Content-Type': 'application/json' }, - }); -} - -// 处理请求并缓存 -const response = await processRequest(request); -await env.KV.put(cacheKey, JSON.stringify(response), { expirationTtl: 3600 }); -``` - -## 📞 支持和资源 - -### 官方文档 -- [Cloudflare Workers 文档](https://developers.cloudflare.com/workers/) -- [Wrangler CLI 文档](https://developers.cloudflare.com/workers/wrangler/) -- [Workers 示例](https://developers.cloudflare.com/workers/examples/) - -### 社区资源 -- [Cloudflare Community](https://community.cloudflare.com/) -- [Workers Discord](https://discord.gg/cloudflare) - -### 调试工具 -```bash -# 本地开发 -wrangler dev - -# 查看日志 -wrangler tail - -# 测试部署 -wrangler deploy --dry-run -``` \ No newline at end of file diff --git a/docs/deployment_checklist.md b/docs/deployment_checklist.md deleted file mode 100644 index 985cf822..00000000 --- a/docs/deployment_checklist.md +++ /dev/null @@ -1,155 +0,0 @@ -# Deployment Checklist - -## Pre-Deployment Checklist - -### Backend Setup -- [ ] **Environment Variables** - - [ ] `OPENAI_API_KEY` is set - - [ ] `FLASK_ENV=production` is set - - [ ] Backend server is accessible from internet - -- [ ] **Server Configuration** - - [ ] Port 8080 is open in firewall - - [ ] SSL certificate is installed (for HTTPS) - - [ ] CORS is configured for frontend domains - - [ ] Server is running on `0.0.0.0:8080` - -- [ ] **Security** - - [ ] API keys are not in version control - - [ ] Server has proper access controls - - [ ] Rate limiting is configured (optional) - -### Frontend Setup -- [ ] **Environment Variables** - - [ ] `VITE_API_BASE_URL` points to backend - - [ ] Environment variables are set in deployment platform - -- [ ] **Build Configuration** - - [ ] Build command: `npm run build` - - [ ] Output directory: `dist` - - [ ] Root directory: `front` - -### Repository Setup -- [ ] **GitHub Secrets** (for GitHub Pages) - - [ ] `VITE_API_BASE_URL` is set in repository secrets - - [ ] GitHub Pages is enabled - - [ ] GitHub Actions workflow is configured - -- [ ] **Cloudflare Pages** (alternative) - - [ ] Repository is connected to Cloudflare Pages - - [ ] Build settings are configured - - [ ] Environment variables are set - -## Deployment Steps - -### 1. Backend Deployment -```bash -# Option A: Local server -./scripts/start_backend.sh - -# Option B: Production server -# Follow the deployment guide for your chosen platform -``` - -### 2. Test Backend -```bash -# Test if backend is accessible -curl -X POST https://your-backend-domain.com:8080/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' -``` - -### 3. Frontend Deployment -- [ ] Push changes to `dev` branch (or `main` branch) -- [ ] Check GitHub Actions workflow (if using GitHub Pages) -- [ ] Verify deployment in Cloudflare Pages dashboard (if using Cloudflare) - -### 4. Test Frontend -- [ ] Open deployed frontend URL -- [ ] Test chat functionality -- [ ] Check browser console for errors -- [ ] Verify API calls are working - -## Post-Deployment Verification - -### Backend Health Check -- [ ] API endpoints respond correctly -- [ ] CORS headers are present -- [ ] SSL certificate is valid -- [ ] Server logs show no errors - -### Frontend Health Check -- [ ] Page loads without errors -- [ ] Chat interface is functional -- [ ] API calls succeed -- [ ] No CORS errors in console - -### Integration Test -- [ ] Send a test message through the chat -- [ ] Verify response is received -- [ ] Check that citations work (if implemented) -- [ ] Test on different browsers - -## Monitoring Setup - -### Backend Monitoring -- [ ] Set up logging -- [ ] Configure error alerts -- [ ] Monitor API usage -- [ ] Track response times - -### Frontend Monitoring -- [ ] Enable analytics -- [ ] Set up error tracking -- [ ] Monitor user interactions -- [ ] Track performance metrics - -## Troubleshooting Common Issues - -### CORS Errors -- [ ] Check if frontend domain is in CORS origins -- [ ] Verify backend is accessible -- [ ] Check SSL certificate validity - -### Connection Issues -- [ ] Verify backend URL is correct -- [ ] Check firewall settings -- [ ] Test with curl or Postman - -### Build Failures -- [ ] Check Node.js version -- [ ] Verify all dependencies are installed -- [ ] Check for TypeScript errors -- [ ] Verify environment variables - -## Security Checklist - -- [ ] API keys are secure -- [ ] HTTPS is enabled -- [ ] CORS is properly configured -- [ ] No sensitive data in logs -- [ ] Rate limiting is in place (optional) -- [ ] Input validation is working - -## Performance Checklist - -- [ ] Frontend loads quickly -- [ ] API responses are fast -- [ ] Images are optimized -- [ ] Bundle size is reasonable -- [ ] Caching is configured - -## Documentation - -- [ ] Update README with deployment instructions -- [ ] Document environment variables -- [ ] Create troubleshooting guide -- [ ] Update API documentation - -## Final Steps - -- [ ] Test with real users -- [ ] Monitor for issues -- [ ] Set up backup procedures -- [ ] Plan for scaling -- [ ] Document lessons learned \ No newline at end of file diff --git a/docs/deployment_guide.md b/docs/deployment_guide.md deleted file mode 100644 index f0d90a02..00000000 --- a/docs/deployment_guide.md +++ /dev/null @@ -1,207 +0,0 @@ -# Deployment Guide: Local Backend with Cloudflare/GitHub Pages Frontend - -## Overview -This guide explains how to deploy the RegHealth Navigator with a local backend server and a frontend hosted on Cloudflare Pages or GitHub Pages. - -> **💡 Quick Start**: For a complete step-by-step setup guide using ngrok + GitHub Pages, see [Local Backend + GitHub Pages Setup](./local_backend_github_pages_setup.md) - -## Architecture -- **Backend**: Flask API running locally (or on a server) -- **Frontend**: React app deployed on Cloudflare Pages or GitHub Pages -- **Connection**: Frontend communicates with backend via HTTPS - -## Step 1: Backend Deployment - -### Option A: Local Development Server -```bash -# Navigate to the backend directory -cd app - -# Install dependencies -pip install -r ../requirements.txt - -# Set environment variables -export OPENAI_API_KEY="your-openai-api-key" -export FLASK_ENV="production" - -# Run the server -python main.py -``` - -### Option B: Production Server (Recommended) -1. **Deploy to a VPS or cloud server** (DigitalOcean, AWS EC2, etc.) -2. **Configure firewall** to allow port 8080 -3. **Set up SSL certificate** (Let's Encrypt) -4. **Use a reverse proxy** (Nginx) for better security - -### Backend Configuration -The backend is configured to accept connections from: -- `https://*.pages.dev` (Cloudflare Pages) -- `https://*.github.io` (GitHub Pages) -- `https://*.cloudflare.com` (Cloudflare Workers) - -## Step 2: Frontend Configuration - -### Environment Variables -Create a `.env.production` file in the `front/` directory: -```bash -# Replace with your actual backend URL -VITE_API_BASE_URL=https://your-backend-domain.com:8080 -``` - -### Build Configuration -The frontend is already configured to use environment variables for the API base URL. - -## Step 3: Frontend Deployment - -### Option A: Cloudflare Pages -1. **Connect your GitHub repository** to Cloudflare Pages -2. **Configure build settings**: - - Build command: `npm run build` - - Build output directory: `dist` - - Root directory: `front` -3. **Set environment variables** in Cloudflare Pages dashboard: - - `VITE_API_BASE_URL`: Your backend URL - -### Option B: GitHub Pages -1. **Enable GitHub Pages** in your repository settings -2. **Set source** to GitHub Actions -3. **Create GitHub Actions workflow** (see below) - -## Step 4: GitHub Actions Workflow (for GitHub Pages) - -The workflow file `.github/workflows/deploy.yml` is already configured to deploy from both `dev` and `main` branches: - -```yaml -name: Deploy to GitHub Pages - -on: - push: - branches: [ dev, main ] - pull_request: - branches: [ dev, main ] - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' - cache: 'npm' - cache-dependency-path: front/package-lock.json - - - name: Install dependencies - run: | - cd front - npm ci - - - name: Build - run: | - cd front - npm run build - env: - VITE_API_BASE_URL: ${{ secrets.VITE_API_BASE_URL }} - - - name: Deploy to GitHub Pages - if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./front/dist -``` - -## Step 5: Security Considerations - -### CORS Configuration -The backend is configured to accept requests from: -- Cloudflare Pages domains -- GitHub Pages domains -- Your specific domains - -### SSL/TLS -- **Backend**: Use Let's Encrypt for free SSL certificates -- **Frontend**: Cloudflare Pages and GitHub Pages provide SSL automatically - -### API Key Security -- Store `OPENAI_API_KEY` as environment variable -- Never commit API keys to version control -- Use secrets management in deployment platforms - -## Step 6: Testing - -### Test Backend -```bash -# Test if backend is accessible -curl -X POST https://your-backend-domain.com:8080/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' -``` - -### Test Frontend -1. Deploy frontend to Cloudflare Pages or GitHub Pages -2. Open the deployed URL -3. Test the chat functionality -4. Check browser console for any CORS errors - -## Troubleshooting - -### Common Issues - -1. **CORS Errors** - - Check if your frontend domain is in the CORS origins list - - Ensure backend is running and accessible - -2. **Connection Refused** - - Check if backend is running on the correct port - - Verify firewall settings - - Check if the backend URL is correct in frontend config - -3. **SSL Certificate Issues** - - Ensure backend has valid SSL certificate - - Check if frontend is using HTTPS - -### Debug Steps -1. Check backend logs for errors -2. Check browser console for frontend errors -3. Test API endpoints directly with curl -4. Verify environment variables are set correctly - -## Monitoring and Maintenance - -### Backend Monitoring -- Set up logging to track API usage -- Monitor server resources -- Set up alerts for downtime - -### Frontend Monitoring -- Use Cloudflare Analytics or GitHub Pages analytics -- Monitor for JavaScript errors -- Track user interactions - -## Cost Considerations - -### Backend Hosting -- VPS: $5-20/month -- Cloud hosting: Varies by provider -- SSL certificates: Free with Let's Encrypt - -### Frontend Hosting -- Cloudflare Pages: Free tier available -- GitHub Pages: Free for public repositories - -### API Costs -- OpenAI API: Pay per use -- Monitor usage to control costs - -## Next Steps - -1. **Set up monitoring** and alerting -2. **Implement rate limiting** on the backend -3. **Add authentication** if needed -4. **Set up CI/CD** for automated deployments -5. **Implement caching** for better performance \ No newline at end of file diff --git a/docs/deployment_scenarios.md b/docs/deployment_scenarios.md deleted file mode 100644 index b871fb63..00000000 --- a/docs/deployment_scenarios.md +++ /dev/null @@ -1,239 +0,0 @@ -# 部署方案详解 - -## 方案 A: 本地后端 + Cloudflare Pages 前端 - -### 架构图 -``` -┌─────────────────┐ HTTP/HTTPS ┌─────────────────────┐ -│ 本地后端 │ ◄──────────────► │ Cloudflare Pages │ -│ localhost:8080 │ │ 前端 (React App) │ -│ 或服务器IP:8080 │ │ │ -└─────────────────┘ └─────────────────────┘ -``` - -### 配置步骤 - -#### 1. 本地后端配置 -```bash -# 启动后端服务器 -./scripts/start_backend.sh - -# 或者手动启动 -cd app -export OPENAI_API_KEY="your-api-key" -export FLASK_ENV="production" -python main.py -``` - -#### 2. Cloudflare Pages 配置 -1. **连接 GitHub 仓库** - - 访问 [Cloudflare Pages](https://pages.cloudflare.com/) - - 点击 "Create a project" - - 选择 "Connect to Git" - - 选择你的 GitHub 仓库 - -2. **构建设置** - ``` - Project name: reghealth-navigator - Production branch: dev - Framework preset: None - Build command: npm run build - Build output directory: dist - Root directory: front - ``` - -3. **环境变量设置** - ``` - VITE_API_BASE_URL = http://your-public-ip:8080 - ``` - *注意：这里需要你的公网IP，不是 localhost* - -#### 3. 网络配置 -```bash -# 检查你的公网IP -curl ifconfig.me - -# 配置防火墙开放8080端口 -# macOS: -sudo pfctl -e -echo "pass in proto tcp from any to any port 8080" | sudo pfctl -f - - -# Linux: -sudo ufw allow 8080 -``` - -### 测试连接 -```bash -# 测试后端是否可以从外网访问 -curl http://your-public-ip:8080/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' -``` - ---- - -## 方案 B: 服务器后端 + Cloudflare Pages 前端 - -### 架构图 -``` -┌─────────────────┐ HTTP/HTTPS ┌─────────────────────┐ -│ 服务器后端 │ ◄──────────────► │ Cloudflare Pages │ -│ your-domain.com │ │ 前端 (React App) │ -│ 或服务器IP:8080 │ │ │ -└─────────────────┘ └─────────────────────┘ -``` - -### 配置步骤 - -#### 1. 服务器部署 -```bash -# 在服务器上部署后端 -git clone your-repo -cd RegHealth-Navigator -./scripts/start_backend.sh -``` - -#### 2. 域名配置 -```bash -# 设置域名解析 -# 在域名提供商处添加 A 记录： -# your-domain.com -> 服务器IP - -# 配置 SSL 证书 -sudo certbot --nginx -d your-domain.com -``` - -#### 3. Cloudflare Pages 环境变量 -``` -VITE_API_BASE_URL = https://your-domain.com:8080 -``` - ---- - -## 方案 C: GitHub Pages + 自定义域名 - -### 架构图 -``` -┌─────────────────┐ HTTP/HTTPS ┌─────────────────────┐ -│ 本地/服务器 │ ◄──────────────► │ GitHub Pages │ -│ 后端 │ │ 前端 (React App) │ -└─────────────────┘ └─────────────────────┘ -``` - -### 配置步骤 - -#### 1. GitHub Pages 设置 -- 进入仓库 Settings → Pages -- Source: "GitHub Actions" -- 自定义域名: `your-domain.com` - -#### 2. 域名解析 -```bash -# 在域名提供商处添加 CNAME 记录： -# your-domain.com -> username.github.io -``` - -#### 3. GitHub Secrets -- 进入 Settings → Secrets and variables → Actions -- 添加: `VITE_API_BASE_URL = https://your-backend-domain.com:8080` - ---- - -## 方案 D: Cloudflare Pages + Cloudflare Workers (推荐) - -### 架构图 -``` -┌─────────────────┐ HTTP/HTTPS ┌─────────────────────┐ -│ Cloudflare │ ◄──────────────► │ Cloudflare Pages │ -│ Workers (后端) │ │ 前端 (React App) │ -└─────────────────┘ └─────────────────────┘ -``` - -### 优势 -- 全托管在 Cloudflare -- 自动 SSL -- 全球 CDN -- 无需管理服务器 - ---- - -## 🔧 网络配置详解 - -### 本地开发测试 -```bash -# 1. 启动后端 -./scripts/start_backend.sh - -# 2. 测试本地连接 -curl http://localhost:8080/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' - -# 3. 获取公网IP -curl ifconfig.me - -# 4. 测试公网访问 -curl http://your-public-ip:8080/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' -``` - -### 端口转发 (如果在内网) -```bash -# 使用 ngrok 进行端口转发 -ngrok http 8080 - -# 或者使用 localtunnel -npx localtunnel --port 8080 -``` - -### CORS 配置检查 -```yaml -# app/config/development.yml -cors: - origins: - - https://*.pages.dev # Cloudflare Pages - - https://*.github.io # GitHub Pages - - https://your-domain.com # 你的自定义域名 -``` - ---- - -## 🚨 常见问题 - -### 1. CORS 错误 -**问题**: 前端无法连接到后端 -**解决**: -- 检查后端 CORS 配置 -- 确保前端域名在后端允许列表中 -- 验证后端 URL 是否正确 - -### 2. 连接被拒绝 -**问题**: 无法从外网访问本地后端 -**解决**: -- 检查防火墙设置 -- 使用端口转发工具 (ngrok, localtunnel) -- 部署到服务器 - -### 3. SSL 证书问题 -**问题**: HTTPS 前端无法连接 HTTP 后端 -**解决**: -- 为后端配置 SSL 证书 -- 使用端口转发工具的 HTTPS 版本 -- 部署到支持 HTTPS 的平台 - ---- - -## 📋 推荐方案 - -### 开发阶段 -- **本地后端** + **Cloudflare Pages 前端** -- 使用 ngrok 进行端口转发 - -### 生产阶段 -- **服务器后端** + **Cloudflare Pages 前端** -- 配置 SSL 证书和域名 - -### 全托管方案 -- **Cloudflare Workers 后端** + **Cloudflare Pages 前端** -- 无需管理服务器 \ No newline at end of file diff --git a/docs/frontend_guide.md b/docs/frontend_guide.md deleted file mode 100644 index a4a4e275..00000000 --- a/docs/frontend_guide.md +++ /dev/null @@ -1,576 +0,0 @@ -# Frontend Guide - -## Overview - -The RegHealth Navigator frontend is built with React, TypeScript, and Tailwind CSS. It provides an intuitive interface for exploring and analyzing Medicare regulations. - -## Architecture - -### Tech Stack -- React 18 -- TypeScript -- Tailwind CSS -- Vite -- Zustand (State Management) -- Lucide Icons - -### Component Structure -``` -Layout -├── Header -│ ├── Logo -│ ├── History Button (Clock) -│ └── Settings Button -├── Tab Navigation -│ ├── Chat -│ ├── Summary -│ ├── FAQ -│ └── Compare -├── Tab Content -│ ├── ChatPanel -│ ├── SummaryTab -│ ├── FAQTab -│ └── ComparisonTab -├── Modals -│ ├── CitationModal -│ └── HistoryModal -└── Document Selector -``` - -## Features - -### 1. Document Management -- **File Naming Convention** - - Format: `{year}_{program_type}_{type}_{document_number}` - - Example: `2024_MPFS_final_2024-14828` - - Program Types: MPFS, HOSPICE, SNF - - Document Types: final, proposed - -- **Document Selection** - - Available in all tabs - - Search with filters (Year, Program, Type) - - Selected file tags with remove option - - Required for content generation - -### 2. Chat Interface -- **Document Context** - - Optional document selection via plus (+) button - - Visual indicators for selected documents - - Chat without documents or with specific files - -- **Message Display** - - User and AI messages - - Citation links - - Code blocks - - Markdown support - -### 3. Citation System -- **Citation Modal** - - Click citation links to open - - Full content display - - Source attribution - - XML parsing simulation - -### 4. History Management -- **History Modal** - - Access via clock icon - - Conversation previews - - Session restoration - - File context preservation - - Message count and preview - -### 5. Comparison View -- **Side-by-side Layout** - - Left/right document comparison - - Color-coded highlighting: - - 🟢 Green: Added content - - 🔴 Red: Removed content - - 🟡 Yellow: Modified content - - Expandable sections - - Change tracking - -## Backend Integration Requirements - -### 1. Document Management API -```typescript -// Document List -GET /api/documents -Response: { - documents: Array<{ - id: string; - name: string; - year: number; - programType: string; - type: string; - documentNumber: string; - }>; -} - -// Document Content -GET /api/documents/{id} -Response: { - content: string; - metadata: { - title: string; - publicationDate: string; - type: string; - }; -} -``` - -### 2. Chat API -```typescript -// Chat Message -POST /api/chat -Request: { - message: string; - documentIds?: string[]; - sessionId?: string; -} -Response: { - message: string; - citations: Citation[]; -} - -// Chat Session -GET /api/chat/sessions -Response: { - sessions: Array<{ - id: string; - name: string; - date: string; - files: string[]; - messageCount: number; - }>; -} -``` - -### 3. Citation API -```typescript -// Citation Details -GET /api/citations/{id} -Response: { - id: string; - title: string; - content: string; - fullContent: string; - documentId: string; - documentName: string; -} -``` - -### 4. Comparison API -```typescript -// Document Comparison -POST /api/compare -Request: { - documentIds: string[]; -} -Response: { - changes: Array<{ - type: 'added' | 'removed' | 'modified'; - content: string; - section: string; - }>; -} -``` - -### 5. Summary & FAQ API -```typescript -// Generate Summary -POST /api/summary -Request: { - documentIds: string[]; -} -Response: { - summary: string; - keyPoints: string[]; -} - -// Generate FAQ -POST /api/faq -Request: { - documentIds: string[]; -} -Response: { - questions: Array<{ - question: string; - answer: string; - citations: Citation[]; - }>; -} -``` - -## State Management - -### Store Structure -```typescript -interface Store { - // UI State - activeTab: 'chat' | 'summary' | 'faq' | 'compare'; - isProcessing: boolean; - processingProgress: number; - - // Modal State - showHistory: boolean; - showCitation: boolean; - showDocumentSelector: boolean; - - // Document State - selectedDocuments: string[]; - availableDocuments: Document[]; - - // Chat State - messages: ChatMessage[]; - currentSession: string | null; - - // History State - sessions: ChatSession[]; -} -``` - -## Implementation Status - -### Completed -- ✅ File naming convention -- ✅ Document selection UI -- ✅ Citation modal -- ✅ History modal -- ✅ Comparison view -- ✅ Chat interface -- ✅ Basic state management - -### Pending Backend Implementation -- ⏳ Document management API -- ⏳ Chat API with document context -- ⏳ Citation system -- ⏳ Comparison API -- ⏳ Summary generation -- ⏳ FAQ generation -- ⏳ History management -- ⏳ Session restoration - -## Development Guidelines - -### Code Style -- Use TypeScript for type safety -- Follow React best practices -- Use Tailwind CSS for styling -- Implement proper error handling -- Add loading states for async operations - -### Testing -- Unit tests for components -- Integration tests for API calls -- E2E tests for critical flows - -### Performance -- Lazy loading for modals -- Virtual scrolling for long lists -- Optimized re-renders -- Proper cleanup in useEffect - -## Project Structure - -``` -front/ -├── src/ -│ ├── components/ # React components -│ │ ├── chat/ # Chat-related components -│ │ ├── layout/ # Layout components (Header, Sidebar, etc.) -│ │ └── results/ # Results panel components -│ ├── store/ # State management (Zustand store) -│ ├── types/ # TypeScript type definitions -│ ├── utils/ # Utility functions -│ ├── App.tsx # Main application component -│ └── main.tsx # Application entry point -├── public/ # Static assets -└── index.html # HTML template -``` - -## Key Components - -### Layout Components -- `Header.tsx`: Top navigation bar with active file display -- `Sidebar.tsx`: Left sidebar with file selection and history -- `Layout.tsx`: Main layout wrapper component - -### Feature Components -- `ChatPanel.tsx`: Chat interface for Q&A -- `ResultsPanel.tsx`: Right panel for displaying summaries and comparisons -- `SummaryTab.tsx`: Summary generation interface -- `FAQTab.tsx`: FAQ generation interface -- `ComparisonTab.tsx`: Document comparison interface - -## Backend Integration - -### API Endpoints - -The frontend interacts with the FastAPI backend through the following endpoints: - -1. **File Management** - ```typescript - // Upload XML file - POST /api/files/upload - Content-Type: multipart/form-data - Body: { file: File } - Response: { id: string, name: string, size: string, date: string } - - // Get file list - GET /api/files - Response: File[] - - // Get file content - GET /api/files/{file_id} - Response: { content: string } - ``` - -2. **Analysis Endpoints** - ```typescript - // Generate summary - POST /api/analyze/summary - Body: { file_ids: string[] } - Response: { sections: Section[] } - - // Generate FAQ - POST /api/analyze/faq - Body: { file_ids: string[] } - Response: { questions: Question[] } - - // Compare documents - POST /api/analyze/compare - Body: { file_ids: string[], aspect: string } - Response: { changes: Change[] } - ``` - -3. **Chat Endpoints** - ```typescript - // Send chat message - POST /api/chat/message - Body: { message: string, file_ids: string[] } - Response: { - content: string, - citations: string[] - } - ``` - -### API Integration Example - -```typescript -// Example of API integration in a component -import axios from 'axios'; - -const API_BASE_URL = 'http://localhost:8000/api'; - -export const uploadFile = async (file: File) => { - const formData = new FormData(); - formData.append('file', file); - - const response = await axios.post(`${API_BASE_URL}/files/upload`, formData, { - headers: { - 'Content-Type': 'multipart/form-data', - }, - }); - - return response.data; -}; - -export const generateSummary = async (fileIds: string[]) => { - const response = await axios.post(`${API_BASE_URL}/analyze/summary`, { - file_ids: fileIds, - }); - - return response.data; -}; -``` - -### Chat Example - -Here's a simple example of how the chat interface interacts with the backend: - -1. **Frontend (ChatPanel.tsx)**: -```typescript -// State for input text -const [input, setInput] = useState(''); - -// Handle form submission -const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - if (input.trim()) { - // Send POST request to backend - const response = await fetch('http://localhost:8000/api/chat', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - section_id: "demo_section", - query: input - }), - }); - - // Handle response - const data = await response.json(); - // Display response in chat - addMessage({ - id: (Date.now() + 1).toString(), - role: 'assistant', - content: data.response - }); - } -}; - -// Input field in JSX - setInput(e.target.value)} - placeholder="Ask about the XML documents..." - className="flex-1 p-2 border border-neutral-300 rounded-md focus:outline-none focus:ring-2 focus:ring-primary-500 focus:border-primary-500" -/> -``` - -2. **Backend (main.py)**: -```python -@app.post("/api/chat") -async def chat(request: SectionRequest): - """Chat with a specific section""" - try: - # Hardcoded response for demonstration - response = f"Hello! You asked about section {request.section_id}. Your query was: {request.query}" - return {"response": response} - except Exception as e: - raise HTTPException(status_code=400, detail=str(e)) -``` - -3. **Data Flow**: - - User types in the input field - - Clicks send button or presses enter - - Frontend sends POST request to `/api/chat` - - Backend processes request and returns response - - Frontend displays response in chat interface - -4. **Testing**: - 1. Start backend server: - ```bash - cd app - uvicorn main:app --reload - ``` - - 2. Start frontend development server: - ```bash - cd front - npm run dev - ``` - - 3. Open frontend in browser - 4. Type message in input field - 5. Click send button - 6. View response in chat interface - -## Error Handling - -Implement error handling for API calls: - -```typescript -try { - const response = await apiCall(); - // Handle success -} catch (error) { - if (axios.isAxiosError(error)) { - // Handle API errors - console.error('API Error:', error.response?.data); - // Show user-friendly error message - } else { - // Handle other errors - console.error('Error:', error); - } -} -``` - -## Development Guidelines - -1. **Component Structure** - - Keep components small and focused - - Use TypeScript interfaces for props - - Implement proper error boundaries - - Use React hooks for state management - -2. **API Integration** - - Use axios for HTTP requests - - Implement proper error handling - - Use environment variables for API URLs - - Implement request/response interceptors for common operations - -3. **State Management** - - Use Zustand for global state - - Keep component state local when possible - - Implement proper loading states - - Handle API errors in the store - -4. **Styling** - - Use Tailwind CSS for styling - - Follow the existing design system - - Maintain responsive design - - Use consistent spacing and colors - -## Environment Setup - -1. Create a `.env` file: - ``` - VITE_API_BASE_URL=http://localhost:8000/api - ``` - -2. Install dependencies: - ```bash - npm install - ``` - -3. Start development server: - ```bash - npm run dev - ``` - -## Testing - -1. Write unit tests for components -2. Test API integration -3. Test error handling -4. Test responsive design - -## Deployment - -1. Build the application: - ```bash - npm run build - ``` - -2. Serve the built files from your FastAPI backend: - ```python - from fastapi.staticfiles import StaticFiles - - app.mount("/", StaticFiles(directory="front/dist", html=True), name="static") - ``` - -## Common Issues and Solutions - -1. **CORS Issues** - - Ensure FastAPI CORS middleware is configured correctly - - Check API URL configuration - -2. **File Upload Issues** - - Check file size limits - - Verify multipart/form-data handling - - Check file type validation - -3. **State Management Issues** - - Verify store updates - - Check component re-renders - - Validate state shape - -## Contributing - -1. Follow the existing code style -2. Write tests for new features -3. Update documentation -4. Create pull requests with clear descriptions \ No newline at end of file diff --git a/docs/frontend_hosting_comparison.md b/docs/frontend_hosting_comparison.md deleted file mode 100644 index 67dacb9c..00000000 --- a/docs/frontend_hosting_comparison.md +++ /dev/null @@ -1,167 +0,0 @@ -# 前端托管平台对比 - -## 🎯 重要说明 - -**你可以选择任意一个平台，不需要同时使用！** - -- ✅ **GitHub Pages** - 完全免费，适合大多数项目 -- ✅ **Cloudflare Pages** - 免费，性能更好，功能更丰富 -- ❌ **不需要同时使用两者** - -## 📊 平台对比 - -| 特性 | GitHub Pages | Cloudflare Pages | -|------|-------------|------------------| -| **免费额度** | 完全免费 | 完全免费 | -| **自定义域名** | ✅ 支持 | ✅ 支持 | -| **SSL 证书** | ✅ 自动 | ✅ 自动 | -| **CDN** | ✅ 基础 | ✅ 全球 CDN | -| **构建速度** | 较慢 | 很快 | -| **环境变量** | ✅ GitHub Secrets | ✅ 内置支持 | -| **分支部署** | ✅ 支持 | ✅ 支持 | -| **预览部署** | ✅ PR 预览 | ✅ PR 预览 | -| **分析工具** | ❌ 无 | ✅ 内置 | -| **边缘函数** | ❌ 无 | ✅ Workers | - -## 🚀 GitHub Pages 部署 - -### 优势 -- **完全免费** - 无任何限制 -- **简单易用** - 配置简单 -- **GitHub 集成** - 与代码仓库完美集成 -- **自动部署** - 推送代码自动部署 - -### 配置步骤 -1. **启用 GitHub Pages** - ```bash - # 进入仓库 Settings → Pages - # Source: "GitHub Actions" - # Branch: 选择你的分支 (dev/main) - ``` - -2. **设置环境变量** - ```bash - # 进入 Settings → Secrets and variables → Actions - # 添加: VITE_API_BASE_URL = https://your-backend-domain.com:8080 - ``` - -3. **推送代码** - ```bash - git push origin dev - # 自动触发部署 - ``` - -### 访问地址 -``` -https://your-username.github.io/your-repo-name -``` - -## ⚡ Cloudflare Pages 部署 - -### 优势 -- **性能更好** - 全球 CDN 加速 -- **功能丰富** - 更多高级功能 -- **分析工具** - 内置访问分析 -- **边缘计算** - 支持 Workers - -### 配置步骤 -1. **连接仓库** - ```bash - # 访问 https://pages.cloudflare.com/ - # 点击 "Create a project" - # 选择 "Connect to Git" - # 选择你的 GitHub 仓库 - ``` - -2. **构建设置** - ``` - Project name: reghealth-navigator - Production branch: dev - Framework preset: None - Build command: npm run build - Build output directory: dist - Root directory: front - ``` - -3. **环境变量** - ``` - VITE_API_BASE_URL = https://your-backend-domain.com:8080 - ``` - -### 访问地址 -``` -https://reghealth-navigator.pages.dev -``` - -## 🔧 后端配置 - -无论选择哪个平台，后端配置都是一样的： - -### CORS 配置 -```yaml -# app/config/development.yml -cors: - origins: - - https://*.github.io # GitHub Pages - - https://*.pages.dev # Cloudflare Pages - - https://your-domain.com # 自定义域名 -``` - -### 环境变量 -```bash -# 后端启动 -export OPENAI_API_KEY="your-api-key" -./scripts/start_backend.sh -``` - -## 📋 推荐选择 - -### 选择 GitHub Pages 如果你： -- ✅ 想要最简单的配置 -- ✅ 预算有限（完全免费） -- ✅ 项目规模较小 -- ✅ 主要使用 GitHub 生态系统 - -### 选择 Cloudflare Pages 如果你： -- ✅ 需要更好的性能 -- ✅ 想要更多高级功能 -- ✅ 计划使用边缘计算 -- ✅ 需要详细的分析数据 - -## 🎯 我的建议 - -### 开发阶段 -**推荐 GitHub Pages** - 配置简单，完全免费 - -### 生产阶段 -**推荐 Cloudflare Pages** - 性能更好，功能更丰富 - -## 🔄 迁移指南 - -### 从 GitHub Pages 迁移到 Cloudflare Pages -1. 在 Cloudflare Pages 创建新项目 -2. 连接相同的 GitHub 仓库 -3. 配置相同的环境变量 -4. 更新 DNS 记录（如果需要自定义域名） -5. 删除 GitHub Pages 配置 - -### 从 Cloudflare Pages 迁移到 GitHub Pages -1. 在 GitHub 启用 Pages -2. 配置 GitHub Actions 工作流 -3. 设置 GitHub Secrets -4. 更新 DNS 记录 -5. 删除 Cloudflare Pages 项目 - -## ❓ 常见问题 - -### Q: 必须使用 Cloudflare 吗？ -**A: 不是！** GitHub Pages 完全可以满足需求。 - -### Q: 哪个平台更好？ -**A: 取决于你的需求** - GitHub Pages 更简单，Cloudflare Pages 功能更丰富。 - -### Q: 可以同时使用两个平台吗？ -**A: 技术上可以，但不推荐** - 会增加维护复杂度。 - -### Q: 如何选择？ -**A: 建议从 GitHub Pages 开始** - 简单易用，完全免费。 \ No newline at end of file diff --git a/docs/no_public_ip_solution.md b/docs/no_public_ip_solution.md deleted file mode 100644 index 4ddaf0cb..00000000 --- a/docs/no_public_ip_solution.md +++ /dev/null @@ -1,254 +0,0 @@ -# 无公网 IP 部署解决方案 - -## 🚨 问题描述 - -你的情况： -- ✅ 本地后端运行正常 -- ❌ 没有公网 IP -- ❌ 前端无法访问本地后端 -- ✅ 前端部署在 GitHub Pages/Cloudflare Pages - -## 🛠️ 解决方案 - -### 方案 1: ngrok (推荐用于开发) - -#### 安装 ngrok -```bash -# macOS -brew install ngrok - -# 或者访问 https://ngrok.com/download -``` - -#### 启动后端 -```bash -# 1. 启动后端 -./scripts/start_backend.sh - -# 2. 在另一个终端启动 ngrok -ngrok http 8080 -``` - -#### ngrok 输出示例 -``` -Session Status online -Account your-email@example.com -Version 3.5.0 -Region United States (us) -Latency 51ms -Web Interface http://127.0.0.1:4040 -Forwarding https://abc123.ngrok.io -> http://localhost:8080 -``` - -#### 配置前端 -在 GitHub Pages 或 Cloudflare Pages 中设置环境变量： -``` -VITE_API_BASE_URL = https://abc123.ngrok.io -``` - -### 方案 2: localtunnel (免费替代) - -#### 安装 localtunnel -```bash -npm install -g localtunnel -``` - -#### 启动隧道 -```bash -# 启动后端 -./scripts/start_backend.sh - -# 启动 localtunnel -lt --port 8080 -``` - -#### 输出示例 -``` -your url is: https://abc123.loca.lt -``` - -#### 配置前端 -``` -VITE_API_BASE_URL = https://abc123.loca.lt -``` - -### 方案 3: 云服务器部署 (生产环境) - -#### 选择云服务器 -- **DigitalOcean**: $5/月 -- **AWS EC2**: 免费层可用 -- **Google Cloud**: 免费层可用 -- **Vultr**: $2.5/月 - -#### 部署步骤 -```bash -# 1. 在服务器上克隆代码 -git clone https://github.com/your-username/RegHealth-Navigator.git -cd RegHealth-Navigator - -# 2. 安装依赖 -pip install -r requirements.txt - -# 3. 设置环境变量 -export OPENAI_API_KEY="your-api-key" - -# 4. 启动后端 -./scripts/start_backend.sh -``` - -#### 配置防火墙 -```bash -# Ubuntu/Debian -sudo ufw allow 8080 - -# CentOS/RHEL -sudo firewall-cmd --permanent --add-port=8080/tcp -sudo firewall-cmd --reload -``` - -#### 配置前端 -``` -VITE_API_BASE_URL = http://your-server-ip:8080 -``` - -### 方案 4: Cloudflare Workers (全托管) - -#### 优势 -- 无需管理服务器 -- 自动 SSL -- 全球 CDN -- 完全免费 - -#### 部署步骤 -1. 将后端代码转换为 Cloudflare Workers -2. 部署到 Cloudflare Workers -3. 前端直接连接 Workers - -## 🧪 测试连接 - -### 使用测试脚本 -```bash -# 测试 ngrok 连接 -python scripts/test_cloudflare_connection.py - -# 测试 localtunnel 连接 -curl https://abc123.loca.lt/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' -``` - -### 手动测试 -```bash -# 测试本地后端 -curl http://localhost:8080/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' - -# 测试 ngrok 隧道 -curl https://abc123.ngrok.io/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' -``` - -## 📋 推荐工作流程 - -### 开发阶段 (推荐 ngrok) -```bash -# 1. 启动后端 -./scripts/start_backend.sh - -# 2. 启动 ngrok (新终端) -ngrok http 8080 - -# 3. 复制 ngrok URL -# 例如: https://abc123.ngrok.io - -# 4. 在 GitHub Pages/Cloudflare Pages 设置 -VITE_API_BASE_URL = https://abc123.ngrok.io - -# 5. 推送代码触发部署 -git push origin dev -``` - -### 生产阶段 (推荐云服务器) -1. 部署后端到云服务器 -2. 配置域名和 SSL -3. 设置环境变量指向服务器 -4. 监控和维护 - -## 🔧 自动化脚本 - -### 启动开发环境 -```bash -#!/bin/bash -# start_dev_with_ngrok.sh - -echo "🚀 Starting development environment..." - -# 启动后端 -./scripts/start_backend.sh & - -# 等待后端启动 -sleep 5 - -# 启动 ngrok -echo "🌐 Starting ngrok tunnel..." -ngrok http 8080 --log=stdout -``` - -### 获取 ngrok URL -```bash -#!/bin/bash -# get_ngrok_url.sh - -echo "🔍 Getting ngrok URL..." - -# 获取 ngrok URL -NGROK_URL=$(curl -s http://localhost:4040/api/tunnels | jq -r '.tunnels[0].public_url') - -if [ "$NGROK_URL" != "null" ]; then - echo "✅ ngrok URL: $NGROK_URL" - echo "📝 Use this URL in your frontend environment variables:" - echo " VITE_API_BASE_URL = $NGROK_URL" -else - echo "❌ ngrok not running" - echo " Start ngrok with: ngrok http 8080" -fi -``` - -## 🚨 注意事项 - -### ngrok 限制 -- **免费版**: 每次重启 URL 会变化 -- **付费版**: 固定域名，更稳定 -- **连接数**: 免费版有连接数限制 - -### 安全考虑 -- ngrok 会暴露你的本地服务到公网 -- 建议只在开发时使用 -- 生产环境使用云服务器 - -### 性能考虑 -- ngrok 会增加延迟 -- 适合开发测试，不适合生产 - -## 💰 成本对比 - -| 方案 | 成本 | 适用场景 | -|------|------|----------| -| ngrok 免费版 | $0 | 开发测试 | -| ngrok 付费版 | $8/月 | 开发+演示 | -| localtunnel | $0 | 开发测试 | -| 云服务器 | $5-20/月 | 生产环境 | -| Cloudflare Workers | $0 | 全托管 | - -## 🎯 推荐选择 - -### 立即开始 (今天) -**使用 ngrok** - 免费，快速，适合开发 - -### 长期方案 (生产) -**云服务器** - 稳定，可控，适合生产 - -### 全托管方案 -**Cloudflare Workers** - 无需管理服务器 \ No newline at end of file diff --git a/docs/port_forwarding_guide.md b/docs/port_forwarding_guide.md deleted file mode 100644 index 13fef816..00000000 --- a/docs/port_forwarding_guide.md +++ /dev/null @@ -1,233 +0,0 @@ -# 端口转发指南 - -## 为什么需要端口转发？ - -当你的前端部署在 Cloudflare Pages 上时，它需要能够访问你的后端 API。如果你的后端运行在本地机器上，Cloudflare Pages 无法直接访问 `localhost:8080`，因为： - -1. **localhost** 只在你的本地机器上有效 -2. **Cloudflare Pages** 运行在 Cloudflare 的服务器上 -3. **跨域访问** 需要公网可访问的地址 - -## 解决方案 - -### 方案 1: ngrok (推荐用于开发) - -#### 安装 ngrok -```bash -# macOS -brew install ngrok - -# 或者下载安装 -# 访问 https://ngrok.com/download -``` - -#### 启动隧道 -```bash -# 启动 HTTP 隧道 -ngrok http 8080 - -# 或者启动 HTTPS 隧道 (推荐) -ngrok http 8080 --scheme https -``` - -#### 输出示例 -``` -Session Status online -Account your-email@example.com -Version 3.5.0 -Region United States (us) -Latency 51ms -Web Interface http://127.0.0.1:4040 -Forwarding https://abc123.ngrok.io -> http://localhost:8080 -``` - -#### 在 Cloudflare Pages 中使用 -``` -VITE_API_BASE_URL = https://abc123.ngrok.io -``` - -### 方案 2: localtunnel - -#### 安装 localtunnel -```bash -npm install -g localtunnel -``` - -#### 启动隧道 -```bash -lt --port 8080 -``` - -#### 输出示例 -``` -your url is: https://abc123.loca.lt -``` - -#### 在 Cloudflare Pages 中使用 -``` -VITE_API_BASE_URL = https://abc123.loca.lt -``` - -### 方案 3: 配置防火墙 (生产环境) - -#### macOS 防火墙配置 -```bash -# 启用防火墙 -sudo pfctl -e - -# 添加规则允许 8080 端口 -echo "pass in proto tcp from any to any port 8080" | sudo pfctl -f - - -# 检查规则 -sudo pfctl -s rules -``` - -#### Linux 防火墙配置 -```bash -# Ubuntu/Debian -sudo ufw allow 8080 - -# CentOS/RHEL -sudo firewall-cmd --permanent --add-port=8080/tcp -sudo firewall-cmd --reload -``` - -#### 获取公网 IP -```bash -curl ifconfig.me -``` - -#### 在 Cloudflare Pages 中使用 -``` -VITE_API_BASE_URL = http://your-public-ip:8080 -``` - -## 测试连接 - -### 使用测试脚本 -```bash -# 测试本地连接 -python scripts/test_deployment.py - -# 测试公网连接 -python scripts/test_cloudflare_connection.py -``` - -### 手动测试 -```bash -# 测试本地后端 -curl http://localhost:8080/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' - -# 测试 ngrok 隧道 -curl https://abc123.ngrok.io/api/simple-chat \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' -``` - -## 安全考虑 - -### ngrok 安全设置 -```bash -# 设置认证令牌 -ngrok config add-authtoken YOUR_AUTH_TOKEN - -# 限制访问 -ngrok http 8080 --basic-auth "username:password" -``` - -### 生产环境建议 -1. **使用 HTTPS** - 确保前端和后端都使用 HTTPS -2. **配置 CORS** - 只允许特定域名访问 -3. **使用域名** - 不要直接暴露 IP 地址 -4. **监控访问** - 设置访问日志和监控 - -## 故障排除 - -### 常见问题 - -#### 1. ngrok 连接失败 -```bash -# 检查 ngrok 状态 -ngrok status - -# 重新启动隧道 -ngrok http 8080 --log=stdout -``` - -#### 2. CORS 错误 -确保后端 CORS 配置包含 ngrok 域名： -```yaml -# app/config/development.yml -cors: - origins: - - https://*.ngrok.io - - https://*.loca.lt - - https://*.pages.dev -``` - -#### 3. 端口被占用 -```bash -# 检查端口占用 -lsof -i :8080 - -# 杀死占用进程 -kill -9 PID -``` - -## 推荐工作流程 - -### 开发阶段 -1. 启动本地后端：`./scripts/start_backend.sh` -2. 启动 ngrok 隧道：`ngrok http 8080` -3. 在 Cloudflare Pages 设置环境变量 -4. 推送代码到 `dev` 分支触发部署 -5. 测试连接 - -### 生产阶段 -1. 部署后端到服务器 -2. 配置域名和 SSL 证书 -3. 在 Cloudflare Pages 使用生产环境 URL -4. 监控和日志记录 - -## 自动化脚本 - -### 启动开发环境 -```bash -#!/bin/bash -# start_dev.sh - -echo "🚀 Starting development environment..." - -# 启动后端 -./scripts/start_backend.sh & - -# 等待后端启动 -sleep 5 - -# 启动 ngrok -ngrok http 8080 --log=stdout -``` - -### 测试连接 -```bash -#!/bin/bash -# test_connection.sh - -echo "🔍 Testing connection..." - -# 获取 ngrok URL -NGROK_URL=$(curl -s http://localhost:4040/api/tunnels | jq -r '.tunnels[0].public_url') - -if [ "$NGROK_URL" != "null" ]; then - echo "✅ ngrok URL: $NGROK_URL" - echo "🌐 Testing API connection..." - - curl -X POST "$NGROK_URL/api/simple-chat" \ - -H "Content-Type: application/json" \ - -d '{"message": "test"}' -else - echo "❌ ngrok not running" -fi -``` \ No newline at end of file diff --git a/docs/quick_deployment_summary.md b/docs/quick_deployment_summary.md deleted file mode 100644 index 90c8a335..00000000 --- a/docs/quick_deployment_summary.md +++ /dev/null @@ -1,126 +0,0 @@ -# Quick Deployment Summary - -## 🚀 One-Minute Setup - -### Backend (Local/Server) -```bash -# 1. Set your OpenAI API key -export OPENAI_API_KEY="your-api-key" - -# 2. Start backend with one command -./scripts/start_backend.sh - -# 3. Test backend -python scripts/test_deployment.py -``` - -### Frontend (Cloudflare Pages) -1. **Connect to Cloudflare Pages** - - Go to [Cloudflare Pages](https://pages.cloudflare.com/) - - Connect your GitHub repository - - Set build settings: - - Build command: `npm run build` - - Build output directory: `dist` - - Root directory: `front` - -2. **Set Environment Variable** - - In Cloudflare Pages dashboard, add: - - `VITE_API_BASE_URL` = `https://your-backend-domain.com:8080` - -3. **Deploy** - - Push to `main` branch - - Cloudflare will auto-deploy - -### Frontend (GitHub Pages) -1. **Enable GitHub Pages** - - Go to repository Settings → Pages - - Set source to "GitHub Actions" - -2. **Set Repository Secret** - - Go to Settings → Secrets and variables → Actions - - Add `VITE_API_BASE_URL` = `https://your-backend-domain.com:8080` - -3. **Deploy** - - Push to `dev` branch (or `main` branch) - - GitHub Actions will auto-deploy - -## 🔧 Configuration Files Updated - -### Backend CORS (app/config/development.yml & production.yml) -```yaml -cors: - origins: - - https://*.pages.dev # Cloudflare Pages - - https://*.github.io # GitHub Pages - - https://*.cloudflare.com # Cloudflare Workers -``` - -### Frontend Config (front/src/config/index.ts) -```typescript -export const config = { - api: { - baseUrl: import.meta.env.VITE_API_BASE_URL || 'http://127.0.0.1:8080', - // ... - } -}; -``` - -## 📋 What You Need - -### Backend Requirements -- [ ] OpenAI API key -- [ ] Server accessible from internet (for production) -- [ ] Port 8080 open -- [ ] SSL certificate (for HTTPS) - -### Frontend Requirements -- [ ] GitHub repository -- [ ] Cloudflare account (for Cloudflare Pages) -- [ ] Backend URL for environment variable - -## 🧪 Testing - -### Test Backend -```bash -# Test local backend -python scripts/test_deployment.py - -# Test remote backend -python scripts/test_deployment.py https://your-backend-domain.com:8080 -``` - -### Test Frontend -1. Open deployed frontend URL -2. Try sending a message in the chat -3. Check browser console for errors - -## 🚨 Common Issues - -### CORS Errors -- Check if your frontend domain is in CORS origins -- Ensure backend is accessible from internet -- Verify SSL certificate is valid - -### Connection Refused -- Check if backend is running on port 8080 -- Verify firewall settings -- Test with: `curl https://your-backend-domain.com:8080/api/simple-chat` - -### Build Failures -- Check Node.js version (18+) -- Verify all dependencies installed -- Check environment variables are set - -## 📞 Support - -- **Backend Issues**: Check `app/main.py` logs -- **Frontend Issues**: Check browser console -- **Deployment Issues**: See [Deployment Guide](deployment_guide.md) -- **Troubleshooting**: See [Deployment Checklist](deployment_checklist.md) - -## 🎯 Next Steps - -1. **Monitor**: Set up logging and alerts -2. **Scale**: Add rate limiting and caching -3. **Secure**: Implement authentication if needed -4. **Optimize**: Monitor performance and optimize \ No newline at end of file diff --git a/front/package-lock.json b/front/package-lock.json index 6de93bb5..5e0a3d34 100644 --- a/front/package-lock.json +++ b/front/package-lock.json @@ -13,15 +13,9 @@ "markmap-view": "^0.14.3", "react": "^18.3.1", "react-dom": "^18.3.1", -<<<<<<< HEAD - "react-markdown": "^9.0.1", - "rehype-raw": "^7.0.0", - "remark-gfm": "^4.0.0", -======= "react-markdown": "^9.1.0", "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.1", ->>>>>>> dev "zustand": "^4.5.2" }, "devDependencies": { diff --git a/front/package.json b/front/package.json index da6b7603..9b5d2b47 100644 --- a/front/package.json +++ b/front/package.json @@ -10,16 +10,6 @@ "preview": "vite preview" }, "dependencies": { -<<<<<<< HEAD - "lucide-react": "^0.344.0", - "react": "^18.3.1", - "react-dom": "^18.3.1", - "d3": "^7.8.5", - "react-markdown": "^9.0.1", - "markmap-view": "^0.14.3", - "rehype-raw": "^7.0.0", - "remark-gfm": "^4.0.0", -======= "d3": "^7.8.5", "lucide-react": "^0.344.0", "markmap-view": "^0.14.3", @@ -28,7 +18,6 @@ "react-markdown": "^9.1.0", "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.1", ->>>>>>> dev "zustand": "^4.5.2" }, "devDependencies": { @@ -48,8 +37,4 @@ "typescript-eslint": "^8.3.0", "vite": "^5.4.2" } -<<<<<<< HEAD -} -======= } ->>>>>>> dev diff --git a/front/src/components/chat/ChatMessage.tsx b/front/src/components/chat/ChatMessage.tsx index 87be09f2..99bd6dca 100644 --- a/front/src/components/chat/ChatMessage.tsx +++ b/front/src/components/chat/ChatMessage.tsx @@ -1,8 +1,4 @@ import React from 'react'; -<<<<<<< HEAD -import { useStore } from '../../store/store'; - -======= import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; import { useStore } from '../../store/store'; @@ -15,87 +11,17 @@ interface SourceInfo { }>; } ->>>>>>> dev interface ChatMessageProps { message: { id: string; role: 'user' | 'assistant' | 'system'; content: string; citations?: string[]; -<<<<<<< HEAD -======= sources?: SourceInfo[]; ->>>>>>> dev }; } const ChatMessage: React.FC = ({ message }) => { -<<<<<<< HEAD - const { citations, setActiveCitation, setShowCitationModal } = useStore(); - const isUser = message.role === 'user'; - - const handleCitationClick = (citationId: string) => { - const citation = citations[citationId]; - if (citation) { - setActiveCitation(citation); - setShowCitationModal(true); - } - }; - - // Function to highlight citations in the message text - const renderContentWithCitations = (text: string, citationList?: string[]) => { - if (!citationList || citationList.length === 0) return text; - - let parts = [text]; - - citationList.forEach(citation => { - const newParts: (string | JSX.Element)[] = []; - - parts.forEach((part, index) => { - if (typeof part === 'string') { - const regex = new RegExp(`\\[${citation.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\]`, 'g'); - const splitParts = part.split(regex); - - for (let i = 0; i < splitParts.length; i++) { - if (i > 0) { - newParts.push( - - ); - } - if (splitParts[i]) { - newParts.push(splitParts[i]); - } - } - } else { - newParts.push(part); - } - }); - - parts = newParts; - }); - - return parts; - }; - - return ( -
-
-
- {renderContentWithCitations(message.content, message.citations)} -
-======= const { setActiveCitation, setShowCitationModal } = useStore(); const isUser = message.role === 'user'; @@ -196,7 +122,6 @@ const ChatMessage: React.FC = ({ message }) => {
)} ->>>>>>> dev ); diff --git a/front/src/components/chat/ChatPanel.tsx b/front/src/components/chat/ChatPanel.tsx index 5640c14b..3d225c11 100644 --- a/front/src/components/chat/ChatPanel.tsx +++ b/front/src/components/chat/ChatPanel.tsx @@ -12,10 +12,7 @@ const ChatPanel: React.FC = () => { selectedFiles, setSelectedFiles, files, -<<<<<<< HEAD -======= fetchFiles, ->>>>>>> dev showDocumentSelector, setShowDocumentSelector, searchTerm, @@ -35,11 +32,7 @@ const ChatPanel: React.FC = () => { const searchRef = useRef(null); const [isLoading, setIsLoading] = useState(false); -<<<<<<< HEAD - const programs = ['MPFS', 'HOSPICE', 'SNF', 'QPP']; -======= const programs = ['MPFS', 'HOSPICE', 'SNF']; ->>>>>>> dev const types = ['final', 'proposed']; const years = ['2024', '2023', '2022', '2021']; @@ -56,14 +49,11 @@ const ChatPanel: React.FC = () => { return matchesSearch && matchesYear && matchesProgram && matchesType; }); -<<<<<<< HEAD -======= // Fetch documents on component mount useEffect(() => { fetchFiles(); }, [fetchFiles]); ->>>>>>> dev // Auto-scroll to bottom when messages change useEffect(() => { if (messagesEndRef.current) { @@ -99,10 +89,6 @@ const ChatPanel: React.FC = () => { // Clear input immediately for better UX const messageToSend = input; setInput(''); -<<<<<<< HEAD - - try { -======= setIsLoading(true); try { @@ -118,18 +104,13 @@ const ChatPanel: React.FC = () => { requestBody.doc_names = selectedDocNames; } ->>>>>>> dev const response = await fetch(`${config.api.baseUrl}${config.api.endpoints.chat}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' }, -<<<<<<< HEAD - body: JSON.stringify({ query: messageToSend }) -======= body: JSON.stringify(requestBody) ->>>>>>> dev }); // Check if the response is ok @@ -156,12 +137,8 @@ const ChatPanel: React.FC = () => { id: (Date.now() + 1).toString(), role: 'assistant' as const, content: data.response || 'No response from server', -<<<<<<< HEAD - citations: data.citations || [] -======= citations: data.citations || [], sources: data.sources || [] ->>>>>>> dev }; addMessage(assistantMessage); @@ -184,11 +161,8 @@ const ChatPanel: React.FC = () => { content: errorMessage, citations: [] }); -<<<<<<< HEAD -======= } finally { setIsLoading(false); ->>>>>>> dev } } }; @@ -216,8 +190,6 @@ const ChatPanel: React.FC = () => { return (
-<<<<<<< HEAD -======= {/* Demo Mode Banner */}
@@ -239,7 +211,6 @@ const ChatPanel: React.FC = () => {
->>>>>>> dev {/* Chat Header */}
@@ -298,24 +269,12 @@ const ChatPanel: React.FC = () => { You can select a document and ask me questions, or ask me directly. For more accurate answers, please specify the year, program, and type in your question.

-<<<<<<< HEAD -

Example questions:

-=======

Example questions (demo data):

->>>>>>> dev
  • • When did the CY 2024 Medicare Physician Fee Schedule (MPFS) Final Rule become effective?
  • • What is the finalized conversion factor for CY 2024, and how does it compare to CY 2023?
  • • Why did CMS implement HCPCS code G2211 in 2024?
-<<<<<<< HEAD -
-
- ) : ( - messages.map(message => ( - - )) -=======

⚠️ Note: These are sample responses for demonstration purposes only.

@@ -341,7 +300,6 @@ const ChatPanel: React.FC = () => {
)} ->>>>>>> dev )}
diff --git a/front/src/components/citation/CitationModal.tsx b/front/src/components/citation/CitationModal.tsx index c46e26c5..110a37c0 100644 --- a/front/src/components/citation/CitationModal.tsx +++ b/front/src/components/citation/CitationModal.tsx @@ -1,11 +1,3 @@ -<<<<<<< HEAD -import React from 'react'; -import { useStore } from '../../store/store'; -import { X, ExternalLink } from 'lucide-react'; - -const CitationModal: React.FC = () => { - const { showCitationModal, setShowCitationModal, activeCitation } = useStore(); -======= import React, { useState, useEffect } from 'react'; import { useStore } from '../../store/store'; import { X, ExternalLink, ChevronLeft, ChevronRight } from 'lucide-react'; @@ -187,7 +179,6 @@ const CitationModal: React.FC = () => { return null; }; ->>>>>>> dev if (!showCitationModal || !activeCitation) { return null; @@ -198,12 +189,6 @@ const CitationModal: React.FC = () => {
{/* Modal Header */}
-<<<<<<< HEAD -
-

{activeCitation.id} - {activeCitation.title}

-

{activeCitation.documentName}

-
-=======

{formatSourceName(activeCitation.documentName)}

{availableSources.length > 1 && ( @@ -233,7 +218,6 @@ const CitationModal: React.FC = () => {
)} ->>>>>>> dev - - {/* Document Selector Dropdown */} - {showDocumentSelector && ( -
-
-

Select Documents

- - {/* Search Input */} -
- setSearchTerm(e.target.value)} - /> - - -
- - {/* Filter Panel */} - {showFilters && ( -
-
-

Filters

- {hasActiveFilters && ( - - )} -
- -
-
- - -
- -
- - -
- -
- - -
-
-
- )} - - {/* File List */} -
- {filteredFiles.length > 0 ? ( -
- {filteredFiles.map((file) => ( -
handleFileSelect(file.id)} - > -
-

{file.name}

-
- {file.size} - {file.date} -
-
- {selectedFiles.includes(file.id) && ( -
- )} -
- ))} -
- ) : ( -
- No documents found -
- )} -
-
-
- )} -
- - {/* Selected Files Tags */} - {selectedFiles.length > 0 && ( -
- {selectedFiles.map(fileId => { - const file = files.find(f => f.id === fileId); - if (!file) return null; - - return ( -
- {file.name} - -
- ); - })} -
- )} -
- - {selectedFiles.length >= 2 && ( - - )} -
- ); - } -======= import { SendHorizontal, GitCompare } from 'lucide-react'; import ChatMessage from '../chat/ChatMessage'; @@ -505,150 +142,10 @@ The final rule demonstrates significant responsiveness to public input while mai setShowCitationModal(true); } }; ->>>>>>> dev return (
{/* Header */} -<<<<<<< HEAD -
-
-
-

{comparison.title}

-
- {comparison.leftDocument} - vs - {comparison.rightDocument} -
-
- -
-
- - {/* Comparison Content */} -
-
- {/* Left Document */} -
-
-

{comparison.leftDocument}

-

Proposed Rule

-
-
- {comparison.sections.map((section) => ( -
-
toggleSection(section.title)} - > -

{section.title}

- - - -
- - {expandedSections.includes(section.title) && ( -
- {section.changes - .filter(change => change.section !== 'right') - .map((change, idx) => ( -
- {renderChangeIcon(change.type)} -
- {change.type === 'modified' && change.oldContent ? ( -

{change.oldContent}

- ) : ( -

- {change.content} -

- )} -
-
- ))} -
- )} -
- ))} -
-
- - {/* Right Document */} -
-
-

{comparison.rightDocument}

-

Final Rule

-
-
- {comparison.sections.map((section) => ( -
-
toggleSection(section.title)} - > -

{section.title}

- - - -
- - {expandedSections.includes(section.title) && ( -
- {section.changes - .filter(change => change.section !== 'left') - .map((change, idx) => ( -
- {renderChangeIcon(change.type)} -
-

- {change.content} -

-
-
- ))} -
- )} -
- ))} -
-
-
-=======
@@ -722,7 +219,6 @@ The final rule demonstrates significant responsiveness to public input while mai ->>>>>>> dev
); diff --git a/front/src/components/results/FAQTab.tsx b/front/src/components/results/FAQTab.tsx deleted file mode 100644 index d1c1f4bd..00000000 --- a/front/src/components/results/FAQTab.tsx +++ /dev/null @@ -1,416 +0,0 @@ -import React, { useState, useRef, useEffect } from 'react'; -import { useStore } from '../../store/store'; -import { RefreshCw, Download, Copy, HelpCircle, Search, Filter, X } from 'lucide-react'; - -const FAQTab: React.FC = () => { - const { - faq, - setFAQ, - setProcessing, - setProcessingProgress, - citations, - setActiveCitation, - setShowCitationModal, - selectedFiles, - setSelectedFiles, - files, - searchTerm, - setSearchTerm, - yearFilter, - setYearFilter, - programFilter, - setProgramFilter, - typeFilter, - setTypeFilter, - showFilters, - setShowFilters - } = useStore(); - - const [expandedSections, setExpandedSections] = useState(['1']); - const [showDocumentSelector, setShowDocumentSelector] = useState(false); - const searchRef = useRef(null); - - const programs = ['MPFS', 'HOSPICE', 'SNF', 'QPP']; - const types = ['final', 'proposed']; - const years = ['2024', '2023', '2022', '2021']; - - const selectedFileNames = selectedFiles - .map(id => files.find(file => file.id === id)?.name || '') - .filter(Boolean); - - // Filter files based on search and filters - const filteredFiles = files.filter(file => { - const matchesSearch = file.name.toLowerCase().includes(searchTerm.toLowerCase()); - const matchesYear = yearFilter === 'all' || file.name.includes(yearFilter); - const matchesProgram = programFilter === 'all' || file.name.includes(programFilter); - const matchesType = typeFilter === 'all' || file.name.includes(typeFilter); - return matchesSearch && matchesYear && matchesProgram && matchesType; - }); - - // Close dropdown when clicking outside - useEffect(() => { - const handleClickOutside = (event: MouseEvent) => { - if (searchRef.current && !searchRef.current.contains(event.target as Node)) { - setShowDocumentSelector(false); - } - }; - - document.addEventListener('mousedown', handleClickOutside); - return () => document.removeEventListener('mousedown', handleClickOutside); - }, []); - - const toggleSection = (sectionId: string) => { - if (expandedSections.includes(sectionId)) { - setExpandedSections(expandedSections.filter(id => id !== sectionId)); - } else { - setExpandedSections([...expandedSections, sectionId]); - } - }; - - const handleGenerateFAQ = () => { - setProcessing(true); - let progress = 0; - - const interval = setInterval(() => { - progress += 5; - setProcessingProgress(progress); - - if (progress >= 100) { - clearInterval(interval); - setProcessing(false); - setProcessingProgress(0); - } - }, 300); - }; - - const handleCopyFAQ = () => { - if (!faq) return; - - let textToCopy = `# ${faq.title}\n\n`; - - faq.sections.forEach(section => { - textToCopy += `## ${section.title}\n`; - section.questions.forEach(qa => { - textToCopy += `Q: ${qa.question}\n`; - textToCopy += `A: ${qa.answer} ${qa.citation}\n\n`; - }); - textToCopy += '\n'; - }); - - navigator.clipboard.writeText(textToCopy); - }; - - const handleCitationClick = (citationId: string) => { - const citation = citations[citationId]; - if (citation) { - setActiveCitation(citation); - setShowCitationModal(true); - } - }; - - const handleFileSelect = (fileId: string) => { - if (selectedFiles.includes(fileId)) { - setSelectedFiles(selectedFiles.filter(id => id !== fileId)); - } else { - setSelectedFiles([...selectedFiles, fileId]); - } - }; - - const removeSelectedFile = (fileId: string) => { - setSelectedFiles(selectedFiles.filter(id => id !== fileId)); - }; - - const clearAllFilters = () => { - setSearchTerm(''); - setYearFilter('all'); - setProgramFilter('all'); - setTypeFilter('all'); - }; - - const hasActiveFilters = searchTerm || yearFilter !== 'all' || programFilter !== 'all' || typeFilter !== 'all'; - - if (!faq || selectedFiles.length === 0) { - return ( -
-
- -
-

Generate FAQ

-

- Select documents and create frequently asked questions with detailed answers. -

- - {/* Document Selection */} -
-
- - - {/* Document Selector Dropdown */} - {showDocumentSelector && ( -
-
-

Select Documents

- - {/* Search Input */} -
- setSearchTerm(e.target.value)} - /> - - -
- - {/* Filter Panel */} - {showFilters && ( -
-
-

Filters

- {hasActiveFilters && ( - - )} -
- -
-
- - -
- -
- - -
- -
- - -
-
-
- )} - - {/* File List */} -
- {filteredFiles.length > 0 ? ( -
- {filteredFiles.map((file) => ( -
handleFileSelect(file.id)} - > -
-

{file.name}

-
- {file.size} - {file.date} -
-
- {selectedFiles.includes(file.id) && ( -
- )} -
- ))} -
- ) : ( -
- No documents found -
- )} -
-
-
- )} -
- - {/* Selected Files Tags */} - {selectedFiles.length > 0 && ( -
- {selectedFiles.map(fileId => { - const file = files.find(f => f.id === fileId); - if (!file) return null; - - return ( -
- {file.name} - -
- ); - })} -
- )} -
- - {selectedFiles.length > 0 && ( - - )} -
- ); - } - - return ( -
- {/* Header */} -
-
-
-

{faq.title}

- {selectedFiles.length > 0 && ( -

- Based on: {selectedFileNames.join(', ')} -

- )} -
-
- - - -
-
-
- - {/* Content */} -
-
- {faq.sections.map((section) => ( -
-
toggleSection(section.id)} - > -

{section.title}

- - - -
- - {expandedSections.includes(section.id) && ( -
-
- {section.questions.map((qa, idx) => ( -
-

- Q: {qa.question} -

-
- A: -
-

{qa.answer}

- -
-
-
- ))} -
-
- )} -
- ))} -
-
-
- ); -}; - -export default FAQTab; \ No newline at end of file diff --git a/front/src/components/results/SummaryTab.tsx b/front/src/components/results/SummaryTab.tsx index 623fb355..c9e40d13 100644 --- a/front/src/components/results/SummaryTab.tsx +++ b/front/src/components/results/SummaryTab.tsx @@ -1,104 +1,3 @@ -<<<<<<< HEAD -import React, { useState, useRef, useEffect } from 'react'; -import { useStore } from '../../store/store'; -import { RefreshCw, Download, Copy, FileText, Search, Filter, X } from 'lucide-react'; - -const SummaryTab: React.FC = () => { - const { - summary, - setSummary, - setProcessing, - setProcessingProgress, - citations, - setActiveCitation, - setShowCitationModal, - selectedFiles, - setSelectedFiles, - files, - searchTerm, - setSearchTerm, - yearFilter, - setYearFilter, - programFilter, - setProgramFilter, - typeFilter, - setTypeFilter, - showFilters, - setShowFilters - } = useStore(); - - const [expandedSections, setExpandedSections] = useState(['1']); - const [showDocumentSelector, setShowDocumentSelector] = useState(false); - const searchRef = useRef(null); - - const programs = ['MPFS', 'HOSPICE', 'SNF', 'QPP']; - const types = ['final', 'proposed']; - const years = ['2024', '2023', '2022', '2021']; - - const selectedFileNames = selectedFiles - .map(id => files.find(file => file.id === id)?.name || '') - .filter(Boolean); - - // Filter files based on search and filters - const filteredFiles = files.filter(file => { - const matchesSearch = file.name.toLowerCase().includes(searchTerm.toLowerCase()); - const matchesYear = yearFilter === 'all' || file.name.includes(yearFilter); - const matchesProgram = programFilter === 'all' || file.name.includes(programFilter); - const matchesType = typeFilter === 'all' || file.name.includes(typeFilter); - return matchesSearch && matchesYear && matchesProgram && matchesType; - }); - - // Close dropdown when clicking outside - useEffect(() => { - const handleClickOutside = (event: MouseEvent) => { - if (searchRef.current && !searchRef.current.contains(event.target as Node)) { - setShowDocumentSelector(false); - } - }; - - document.addEventListener('mousedown', handleClickOutside); - return () => document.removeEventListener('mousedown', handleClickOutside); - }, []); - - const toggleSection = (sectionId: string) => { - if (expandedSections.includes(sectionId)) { - setExpandedSections(expandedSections.filter(id => id !== sectionId)); - } else { - setExpandedSections([...expandedSections, sectionId]); - } - }; - - const handleGenerateSummary = () => { - setProcessing(true); - let progress = 0; - - const interval = setInterval(() => { - progress += 5; - setProcessingProgress(progress); - - if (progress >= 100) { - clearInterval(interval); - setProcessing(false); - setProcessingProgress(0); - } - }, 300); - }; - - const handleCopySummary = () => { - if (!summary) return; - - let textToCopy = `# ${summary.title}\n\n`; - - summary.sections.forEach(section => { - textToCopy += `## ${section.title}\n`; - section.bullets.forEach(bullet => { - textToCopy += `- ${bullet.content} ${bullet.citation}\n`; - }); - textToCopy += '\n'; - }); - - navigator.clipboard.writeText(textToCopy); -======= import React, { useState, useEffect } from 'react'; import { useStore } from '../../store/store'; import { RefreshCw, Download, Copy, FileText, ChevronLeft, ChevronRight } from 'lucide-react'; @@ -223,7 +122,6 @@ const SummaryTab: React.FC = () => { if (selectedSummary?.content) { navigator.clipboard.writeText(selectedSummary.content); } ->>>>>>> dev }; const handleCitationClick = (citationId: string) => { @@ -234,303 +132,6 @@ const SummaryTab: React.FC = () => { } }; -<<<<<<< HEAD - const handleFileSelect = (fileId: string) => { - if (selectedFiles.includes(fileId)) { - setSelectedFiles(selectedFiles.filter(id => id !== fileId)); - } else { - setSelectedFiles([...selectedFiles, fileId]); - } - }; - - const removeSelectedFile = (fileId: string) => { - setSelectedFiles(selectedFiles.filter(id => id !== fileId)); - }; - - const clearAllFilters = () => { - setSearchTerm(''); - setYearFilter('all'); - setProgramFilter('all'); - setTypeFilter('all'); - }; - - const hasActiveFilters = searchTerm || yearFilter !== 'all' || programFilter !== 'all' || typeFilter !== 'all'; - - if (!summary || selectedFiles.length === 0) { - return ( -
-
- -
-

Generate Summary

-

- Select documents and create a comprehensive summary with citations to source material. -

- - {/* Document Selection */} -
-
- - - {/* Document Selector Dropdown */} - {showDocumentSelector && ( -
-
-

Select Documents

- - {/* Search Input */} -
- setSearchTerm(e.target.value)} - /> - - -
- - {/* Filter Panel */} - {showFilters && ( -
-
-

Filters

- {hasActiveFilters && ( - - )} -
- -
-
- - -
- -
- - -
- -
- - -
-
-
- )} - - {/* File List */} -
- {filteredFiles.length > 0 ? ( -
- {filteredFiles.map((file) => ( -
handleFileSelect(file.id)} - > -
-

{file.name}

-
- {file.size} - {file.date} -
-
- {selectedFiles.includes(file.id) && ( -
- )} -
- ))} -
- ) : ( -
- No documents found -
- )} -
-
-
- )} -
- - {/* Selected Files Tags */} - {selectedFiles.length > 0 && ( -
- {selectedFiles.map(fileId => { - const file = files.find(f => f.id === fileId); - if (!file) return null; - - return ( -
- {file.name} - -
- ); - })} -
- )} -
- - {selectedFiles.length > 0 && ( - - )} -
- ); - } - - return ( -
- {/* Header */} -
-
-
-

{summary.title}

- {selectedFiles.length > 0 && ( -

- Based on: {selectedFileNames.join(', ')} -

- )} -
-
- - - -
-
-
- - {/* Content */} -
-
- {summary.sections.map((section) => ( -
-
toggleSection(section.id)} - > -

{section.title}

- - - -
- - {expandedSections.includes(section.id) && ( -
-
    - {section.bullets.map((bullet) => ( -
  • - • -
    - {bullet.content} - -
    -
    • - ))} -
-
- )} -
- ))} -
-======= const goToPage = (page: number) => { setCurrentPage(page); }; @@ -775,7 +376,6 @@ const SummaryTab: React.FC = () => {

)} ->>>>>>> dev
); diff --git a/front/src/config/index.ts b/front/src/config/index.ts index d2fa2c06..8a55a735 100644 --- a/front/src/config/index.ts +++ b/front/src/config/index.ts @@ -6,15 +6,11 @@ export const config = { list: '/api/summarize/list', generate: '/api/summarize' }, -<<<<<<< HEAD - chat: '/api/chat' -======= chat: '/api/chat', documents: '/api/documents', getSummary: '/api/get-summary', availableSummaries: '/api/available-summaries' ->>>>>>> dev } } }; diff --git a/front/src/store/store.ts b/front/src/store/store.ts index 28ba4052..2e4a269b 100644 --- a/front/src/store/store.ts +++ b/front/src/store/store.ts @@ -1,8 +1,5 @@ import { create } from 'zustand'; -<<<<<<< HEAD -======= import config from '../config'; ->>>>>>> dev // Sample data with new naming convention const sampleFiles = [ @@ -10,10 +7,6 @@ const sampleFiles = [ { id: '2', name: '2024_MPFS_proposed_2024-12345', size: '187 MB', date: '2024-05-15' }, { id: '3', name: '2024_HOSPICE_final_2024-15678', size: '92 MB', date: '2024-10-30' }, { id: '4', name: '2023_SNF_final_2023-23456', size: '134 MB', date: '2023-11-15' }, -<<<<<<< HEAD - { id: '5', name: '2024_QPP_proposed_2024-16789', size: '98 MB', date: '2024-06-20' }, -======= ->>>>>>> dev ]; const sampleHistory = [ @@ -107,47 +100,7 @@ const sampleSummary = { ] }; -<<<<<<< HEAD -const sampleFAQ = { - title: '2024 MPFS Final Rule - FAQ', - sections: [ - { - id: '1', - title: 'Payment Changes', - questions: [ - { - question: 'What is the conversion factor for 2024?', - answer: 'The 2024 conversion factor is $32.75, representing a 3.4% decrease from 2023.', - citation: '§2.1' - }, - { - question: 'How do the new E/M payment rates affect my practice?', - answer: 'The new E/M payment methodology simplifies billing and may increase payments for complex visits while standardizing payments for routine visits.', - citation: '§2.3' - } - ] - }, - { - id: '2', - title: 'Quality Reporting', - questions: [ - { - question: 'What are the key MIPS changes for 2024?', - answer: 'The performance threshold has increased to 82.5 points, and there are 12 new quality measures focused on chronic care management.', - citation: '§3.1' - }, - { - question: 'Are there new reporting requirements?', - answer: 'Yes, clinicians must now report on at least two high-priority measures and demonstrate improvement activities.', - citation: '§3.2' - } - ] - } - ] -}; -======= ->>>>>>> dev // Sample citations data const sampleCitations = { @@ -190,12 +143,9 @@ type FileType = { name: string; size: string; date: string; -<<<<<<< HEAD -======= program?: string; year?: string; type?: string; ->>>>>>> dev }; type HistoryItem = { @@ -230,26 +180,7 @@ type Summary = { sections: SummarySection[]; }; -<<<<<<< HEAD -type FAQQuestion = { - question: string; - answer: string; - citation: string; -}; - -type FAQSection = { - id: string; - title: string; - questions: FAQQuestion[]; -}; - -type FAQ = { - title: string; - sections: FAQSection[]; -}; -======= ->>>>>>> dev type Citation = { id: string; @@ -288,11 +219,8 @@ type StoreState = { files: FileType[]; addFile: (file: FileType) => void; -<<<<<<< HEAD -======= setFiles: (files: FileType[]) => void; fetchFiles: () => Promise; ->>>>>>> dev history: HistoryItem[]; addHistoryItem: (item: HistoryItem) => void; @@ -308,12 +236,6 @@ type StoreState = { summary: Summary | null; setSummary: (summary: Summary | null) => void; -<<<<<<< HEAD - faq: FAQ | null; - setFAQ: (faq: FAQ | null) => void; - -======= ->>>>>>> dev comparison: Comparison | null; setComparison: (comparison: Comparison | null) => void; @@ -359,24 +281,6 @@ type StoreState = { }; export const useStore = create((set, get) => ({ -<<<<<<< HEAD - isLoading: false, - setLoading: (loading) => set({ isLoading: loading }), - - activeTab: 'chat', - setActiveTab: (tab) => set({ activeTab: tab }), - - files: sampleFiles, - addFile: (file) => set((state) => ({ files: [...state.files, file] })), - - history: sampleHistory, - addHistoryItem: (item) => set((state) => ({ history: [item, ...state.history] })), - - selectedFiles: [], - setSelectedFiles: (fileIds) => set({ selectedFiles: fileIds }), - - // messages: sampleChat, -======= // Loading state isLoading: false, setLoading: (loading) => set({ isLoading: loading }), @@ -431,60 +335,10 @@ export const useStore = create((set, get) => ({ setSelectedFiles: (fileIds) => set({ selectedFiles: fileIds }), // Chat messages ->>>>>>> dev messages: [], addMessage: (message) => set((state) => ({ messages: [...state.messages, message] })), setMessages: (messages) => set({ messages }), clearMessages: () => set({ messages: [] }), -<<<<<<< HEAD - - summary: sampleSummary, - setSummary: (summary) => set({ summary }), - - faq: sampleFAQ, - setFAQ: (faq) => set({ faq }), - - comparison: null, - setComparison: (comparison) => set({ comparison }), - - citations: sampleCitations, - setCitations: (citations) => set({ citations }), - - activeCitation: null, - setActiveCitation: (citation) => set({ activeCitation: citation }), - - isProcessing: false, - setProcessing: (processing) => set({ isProcessing: processing }), - - processingProgress: 0, - setProcessingProgress: (progress) => set({ processingProgress: progress }), - - // Search and filter state - searchTerm: '', - setSearchTerm: (term) => set({ searchTerm: term }), - - yearFilter: 'all', - setYearFilter: (year) => set({ yearFilter: year }), - - programFilter: 'all', - setProgramFilter: (program) => set({ programFilter: program }), - - typeFilter: 'all', - setTypeFilter: (type) => set({ typeFilter: type }), - - showFilters: false, - setShowFilters: (show) => set({ showFilters: show }), - - // History modal - showHistory: false, - setShowHistory: (show) => set({ showHistory: show }), - - // Citation modal - showCitationModal: false, - setShowCitationModal: (show) => set({ showCitationModal: show }), - - // Document selection for chat -======= // Summary summary: sampleSummary, @@ -534,7 +388,6 @@ export const useStore = create((set, get) => ({ setShowCitationModal: (show) => set({ showCitationModal: show }), // Document selector ->>>>>>> dev showDocumentSelector: false, setShowDocumentSelector: (show) => set({ showDocumentSelector: show }), })); \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 27e92906..29e739c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,3 @@ -<<<<<<< HEAD -flask==3.1.1 -flask-cors==6.0.1 -python-dotenv==1.0.1 -openai==1.12.0 -faiss-cpu==1.11.0 -======= # Core Flask and Web Framework flask==3.1.1 flask-cors==6.0.1 @@ -16,20 +9,11 @@ PyYAML==6.0.2 # AI and Language Models openai==1.95.1 ->>>>>>> dev langchain==0.1.12 langchain-community==0.0.38 langchain-core==0.1.53 langchain-openai==0.0.8 langchain-text-splitters==0.0.2 -<<<<<<< HEAD -pydantic==2.6.1 -PyYAML==6.0.2 -requests==2.31.0 -tiktoken==0.6.0 -numpy==1.26.4 -pandas==2.2.3 -======= langgraph==0.0.27 langsmith==0.1.22 @@ -102,4 +86,3 @@ typing_extensions==4.13.2 typing-inspect==0.9.0 typing-inspection==0.4.0 annotated-types==0.7.0 ->>>>>>> dev diff --git a/scripts/README_xml_analysis.md b/scripts/README_xml_analysis.md deleted file mode 100644 index 0e743152..00000000 --- a/scripts/README_xml_analysis.md +++ /dev/null @@ -1,127 +0,0 @@ -# XML Label Analysis Scripts - -This directory contains scripts for analyzing XML files in the data directory and comparing XML label types across different folders (HOSPICE, MPFS, SNF). - -## Scripts Overview - -### 1. `analyze_xml_labels.py` - Comprehensive Analysis Script - -**Purpose**: Performs a detailed analysis of XML labels across all folders in the data directory. - -**Features**: -- Analyzes all XML files in HOSPICE, MPFS, and SNF folders -- Extracts all unique XML labels (tag names) from each file -- Compares labels across folders to check for consistency -- Generates detailed reports and saves results to JSON -- Provides comprehensive statistics and analysis - -**Usage**: -```bash -python scripts/analyze_xml_labels.py -``` - -**Output**: -- Console report with detailed analysis -- JSON file: `xml_label_analysis_results.json` -- Summary indicating whether folders use consistent XML labels - -### 2. `quick_xml_label_check.py` - Quick Check Script - -**Purpose**: Provides a concise summary of XML label consistency across folders. - -**Features**: -- Fast analysis of XML label consistency -- Simple console output with key findings -- No file output, just immediate results - -**Usage**: -```bash -python scripts/quick_xml_label_check.py -``` - -**Output**: -- Quick summary of label counts per folder -- Whether all folders use the same label set -- List of unique labels per folder (if any differences found) -- List of common labels across all folders - -### 3. `test_xml_label_analyzer.py` - Unit Tests - -**Purpose**: Unit tests to verify the functionality of the XML label analyzer. - -**Features**: -- Tests all major functions of the analyzer -- Uses temporary test files -- Verifies edge cases (empty folders, nonexistent folders) -- Ensures correct label extraction and comparison - -**Usage**: -```bash -python scripts/test_xml_label_analyzer.py -``` - -## Analysis Results Summary - -Based on the analysis of the current data directory: - -### Label Counts by Folder Type: -- **HOSPICE**: 52 unique XML labels -- **MPFS**: 56 unique XML labels -- **SNF**: 51 unique XML labels - -### Consistency Check: -❌ **INCONSISTENT** - Folders have different XML label sets - -### Differences Found: -- **MPFS** has 4 unique labels not found in other folders: - - `APPENDIX` - - `CONTENTS` - - `NOTE` - - `SECHD` - -### Common Labels: -All three folder types share 49 common XML labels, including: -- `RULE`, `PREAMB`, `AGENCY`, `SUBJECT` -- `P`, `HD`, `SECTION`, `SECTNO` -- `FTNT`, `FTREF`, `SU`, `E` -- And 39 other common labels - -## Key Findings - -1. **High Consistency**: 49 out of 52-56 labels are shared across all folder types -2. **Minor Differences**: Only MPFS has additional labels, likely for specific document features -3. **Core Structure**: All folders use the same core XML structure for regulatory documents -4. **Processing Compatibility**: The differences are minor enough that the same XML processing logic can handle all folder types - -## Technical Details - -### XML Label Categories: -- **Document Structure**: `RULE`, `PREAMB`, `SUPLINF` -- **Metadata**: `AGENCY`, `SUBJECT`, `CFR`, `DEPDOC`, `RIN` -- **Content**: `P`, `HD`, `SECTION`, `SECTNO` -- **Formatting**: `E`, `SU`, `FTNT`, `FTREF` -- **Tables**: `GPOTABLE`, `ROW`, `CHED` -- **MPFS-specific**: `APPENDIX`, `CONTENTS`, `NOTE`, `SECHD` - -### Error Handling: -- Scripts handle malformed XML files gracefully -- Empty or nonexistent folders are handled appropriately -- Detailed logging for debugging - -## Usage Recommendations - -1. **For Quick Checks**: Use `quick_xml_label_check.py` -2. **For Detailed Analysis**: Use `analyze_xml_labels.py` -3. **For Development**: Run `test_xml_label_analyzer.py` to verify functionality -4. **For Integration**: Import `XMLLabelAnalyzer` class from `analyze_xml_labels.py` - -## Dependencies - -- Python 3.6+ -- Standard library modules: `xml.etree.ElementTree`, `pathlib`, `json`, `logging` -- No external dependencies required - -## Author - -Fanxing Bu -Date: 2024-12-19 \ No newline at end of file diff --git a/scripts/XML_LABEL_ANALYSIS_SUMMARY.md b/scripts/XML_LABEL_ANALYSIS_SUMMARY.md deleted file mode 100644 index c9012817..00000000 --- a/scripts/XML_LABEL_ANALYSIS_SUMMARY.md +++ /dev/null @@ -1,251 +0,0 @@ -# XML标签分析总结与优化方案 - -## 📊 分析结果总览 - -### 标签统计 -- **HOSPICE**: 52个唯一XML标签 -- **MPFS**: 56个唯一XML标签 -- **SNF**: 51个唯一XML标签 -- **共有标签**: 49个 -- **MPFS独有标签**: 4个 (APPENDIX, CONTENTS, NOTE, SECHD) - -### 一致性评估 -❌ **INCONSISTENT** - 文件夹间存在标签差异，但核心结构高度一致 - ---- - -## 🏷️ 完整XML标签列表 - -### 🔗 所有文件夹共有的标签 (49个) -``` -ACT, ADD, AGENCY, AGY, AMDPAR, AUTH, BILCOD, BOXHD, CFR, CHED, DATED, DATES, DEPDOC, E, EFFDATE, ENT, FP, FRDOC, FTNT, FTREF, FURINF, GID, GPH, GPOTABLE, HD, LI, LSTSUB, NAME, P, PART, PREAMB, PRORULE, PRTPAGE, REGTEXT, RIN, ROW, RULE, SECTION, SECTNO, SIG, STARS, SU, SUBAGY, SUBJECT, SUM, SUPLINF, TITLE, TNOTE, TTITLE -``` - -### 📁 各文件夹独有标签 - -#### HOSPICE 文件夹 (52个标签) -**独有标签**: 无 (0个) -**额外标签**: `FR`, `SUBPART`, `TDESC` - -#### MPFS 文件夹 (56个标签) -**独有标签**: 4个 -- `APPENDIX` - 附录 -- `CONTENTS` - 目录 -- `NOTE` - 注释 -- `SECHD` - 章节标题 - -**额外标签**: `EXTRACT`, `FR`, `SUBPART` - -#### SNF 文件夹 (51个标签) -**独有标签**: 无 (0个) -**额外标签**: `EXTRACT`, `TDESC` - ---- - -## 🎯 基于语义分析的标签分类 - -### ✅ 建议纳入Chunk的标签 - -#### 主要内容标签 -- **`P`** - 正文段落 (主要内容来源) -- **`E`** - 强调字段/子标题 (保留在段落中) -- **`SU`** - 上标引用 (脚注引用) -- **`FTNT`** - 脚注内容 (重要补充信息) -- **`SUMMARY`** - 摘要内容 (重要概述) -- **`SUBJECT`** - 主题标题 (可作为section_header) -- **`DATES`** - 日期信息 (重要时间信息) -- **`HD`** - 标题 (结构化的标题信息) -- **`SECTION`** - 章节 (重要结构信息) -- **`SECTNO`** - 章节编号 (重要标识) - -#### 格式化标签 -- **`LI`** - 列表项 -- **`LSTSUB`** - 列表子项 -- **`ROW`** - 表格行 -- **`GPOTABLE`** - 政府印刷办公室表格 -- **`TTITLE`** - 表格标题 -- **`CHED`** - 表格列标题 -- **`BOXHD`** - 表格框标题 -- **`TDESC`** - 表格描述 -- **`TNOTE`** - 表格注释 - -### 📋 建议用于Metadata的标签 - -#### 文档信息 -- **`AGENCY`** - 机构信息 -- **`SUBAGY`** - 子机构 -- **`CFR`** - 联邦法规代码 -- **`DEPDOC`** - 部门文档编号 -- **`RIN`** - 法规信息编号 -- **`ACT`** - 法案信息 -- **`AUTH`** - 授权依据 - -#### 签署信息 -- **`SIG`** - 签署信息容器 -- **`NAME`** - 签署人姓名 -- **`TITLE`** - 签署人职称 - -#### 文档标识 -- **`FRDOC`** - 联邦注册文档编号 -- **`BILCOD`** - 账单代码 -- **`GID`** - 图形ID -- **`PRTPAGE`** - 页码信息 - -#### 结构容器 -- **`AMDPAR`** - 修订段落引导语 -- **`PRORULE`** - 拟议规则 -- **`REGTEXT`** - 法规文本容器 -- **`PREAMB`** - 前言容器 -- **`SUPLINF`** - 补充信息容器 -- **`RULE`** - 规则容器 - -### ❌ 建议忽略的标签 - -#### 纯格式化标签 -- **`PRTPAGE`** - 页码标记 (仅表示分页，不具语义) -- **`GPH`** - 图形页容器 (页脚图页索引，与chunk内容无关) -- **`STARS`** - 分隔符 (格式标记，不纳入chunk内容) -- **`FTREF`** - 脚注引用标记 (仅格式化) - ---- - -## 🧠 特殊处理标签说明 - -### 重要语义标签 -| 标签 | 用途 | 处理建议 | -|------|------|----------| -| `...` | 标注小节内的小标题或关键术语 | 保留在chunk中，提取T属性 | -| `

+ ` 组合 | 法规正文中小节内的结构性语言 | 保留完整结构 | -| `` | 授权依据 | 放入metadata，不入chunk | -| `` | 补充材料 | 通常不纳入chunk | - -### 层级构建标签 -| 标签 | 层级 | 用途 | -|------|------|------| -| `` | 1 | 主标题 | -| `` | 2 | 一级标题 | -| `` | 3 | 二级标题 | -| `` | 4 | 三级标题 | - ---- - -## 🏗️ 优化Chunk生成策略 - -### 1. 摘要Chunk (chunk_index=0) -**内容来源**: -- `PREAMB > SUMMARY` 段落 -- `PREAMB > SUBJECT` 主题 -- `PREAMB > DATES` 日期信息 - -**格式**: -``` -**主题**: [SUBJECT内容] -**摘要**: [SUMMARY内容] -**日期**: [DATES内容] -``` - -### 2. 结构化Chunk生成 -**基于**: `REGTEXT > SECTION > P` - -**Section Header构建**: -``` -42 CFR Part 600 > § 600.125 > Submission of revisions -``` - -**Chunk内容优先级**: -1. `

` 段落 (主要内容) -2. `` 强调字段 (子标题) -3. `` 标题 (结构信息) -4. `` 章节编号 (标识信息) -5. `` 脚注 (补充信息) - -### 3. 属性提取策略 -| 标签 | 提取属性 | 用途 | -|------|----------|------| -| `HD` | `SOURCE` | 层级判断 | -| `E` | `T` | 格式化类型 | -| `SECTION` | `TITLE`, `PART` | 章节信息 | -| `P` | `ID` | 段落标识 | -| `FTNT` | `ID` | 脚注标识 | - ---- - -## 📈 实施建议 - -### 1. 配置驱动 -- 使用 `xml_labels_config.py` 管理标签分类 -- 支持自定义标签处理策略 -- 便于维护和扩展 - -### 2. 模块化设计 -- 分离标签分类、文本提取、chunk生成逻辑 -- 支持不同文档类型的定制化处理 -- 便于单元测试和调试 - -### 3. 质量保证 -- 保留重要格式化信息 (`**强调**`, `[引用]`) -- 构建清晰的section_header层级 -- 生成有意义的metadata - -### 4. 性能优化 -- 忽略无语义价值的标签 -- 优化文本提取算法 -- 支持增量处理 - ---- - -## 🔧 技术实现 - -### 文件结构 -``` -app/ -├── config/ -│ └── xml_labels_config.py # 标签配置 -├── core/ -│ ├── xml_chunker.py # 原始chunker -│ └── optimized_xml_chunker.py # 优化chunker -└── scripts/ - ├── analyze_xml_labels.py # 标签分析 - ├── quick_xml_label_check.py # 快速检查 - └── simple_chunker_test.py # 功能测试 -``` - -### 核心功能 -1. **标签分类**: 自动识别标签类型和处理策略 -2. **文本提取**: 保留语义价值的格式化信息 -3. **结构构建**: 生成清晰的文档层级结构 -4. **Chunk生成**: 基于语义的智能分块 -5. **Metadata提取**: 自动提取文档元信息 - ---- - -## 📊 预期效果 - -### 质量提升 -- **语义完整性**: 保留重要内容和结构 -- **可读性**: 清晰的格式化输出 -- **一致性**: 统一的处理标准 - -### 效率提升 -- **处理速度**: 忽略无价值标签 -- **存储优化**: 减少冗余信息 -- **维护性**: 配置驱动的灵活管理 - -### 用户体验 -- **搜索质量**: 更好的语义匹配 -- **导航体验**: 清晰的文档结构 -- **内容理解**: 保留关键上下文 - ---- - -## 🎯 下一步计划 - -1. **实施优化chunker**: 基于配置的智能处理 -2. **性能测试**: 验证处理效率和输出质量 -3. **用户反馈**: 收集使用体验和改进建议 -4. **持续优化**: 根据实际使用情况调整策略 - ---- - -*Author: Fanxing Bu* -*Date: 2024-12-19* \ No newline at end of file diff --git a/scripts/fix_filenames_manually.py b/scripts/fix_filenames_manually.py deleted file mode 100644 index 2d53a6b6..00000000 --- a/scripts/fix_filenames_manually.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -""" -Manually fix filenames to keep doc ID unchanged but correct the year. -""" - -import os -import shutil -from pathlib import Path - -# Mapping of current wrong names to correct names (keeping doc ID) -CORRECT_NAMES = { - # MPFS - extracted year: 2025, 2024 - "2025_MPFS_final_2025-2025.xml": "2025_MPFS_final_2025-08676.xml", # was 2026_MPFS_final_2026-08676.xml - "2024_MPFS_final_2024-2024.xml": "2024_MPFS_final_2024-02705.xml", # was 2025_MPFS_final_2025-02705.xml - "2025_MPFS_proposed_2025-2025.xml": "2025_MPFS_proposed_2025-14828.xml", # was 2025_MPFS_proposed_2025-14828.xml - "2024_MPFS_final_2024-2024.xml": "2024_MPFS_final_2024-24184.xml", # was 2024_MPFS_final_2024-24184.xml - "2025_MPFS_final_2025-2025.xml": "2025_MPFS_final_2025-25382.xml", # was 2025_MPFS_final_2025-25382.xml - "2024_MPFS_proposed_2024-2024.xml": "2024_MPFS_proposed_2024-14624.xml", # was 2024_MPFS_proposed_2024-14624.xml - - # HOSPICE - extracted year: 2024, 2025, 2026 - "2024_HOSPICE_final_2024-2024.xml": "2024_HOSPICE_final_2024-16116.xml", # was 2024_HOSPICE_final_2024-16116.xml - "2025_HOSPICE_final_2025-2025.xml": "2025_HOSPICE_final_2025-16910.xml", # was 2025_HOSPICE_final_2025-16910.xml - "2025_HOSPICE_final_2025-2025.xml": "2025_HOSPICE_final_2025-22495.xml", # was 2025_HOSPICE_final_2025-22495.xml - "2025_HOSPICE_proposed_2025-2025.xml": "2025_HOSPICE_proposed_2025-06921.xml", # was 2025_HOSPICE_proposed_2025-06921.xml - "2026_HOSPICE_proposed_2026-2026.xml": "2026_HOSPICE_proposed_2026-06317.xml", # was 2026_HOSPICE_proposed_2026-06317.xml - - # SNF - extracted year: 2024, 2025, 2026 - "2024_SNF_final_2024-2024.xml": "2024_SNF_final_2024-07522.xml", # was 2025_SNF_final_2025-07522.xml - "2025_SNF_proposed_2025-2025.xml": "2025_SNF_proposed_2025-06812.xml", # was 2025_SNF_proposed_2025-06812.xml - "2025_SNF_final_2025-2025.xml": "2025_SNF_final_2025-22504.xml", # was 2025_SNF_final_2025-22504.xml - "2024_SNF_final_2024-2024.xml": "2024_SNF_final_2024-16249.xml", # was 2024_SNF_final_2024-16249.xml - "2026_SNF_proposed_2026-2026.xml": "2026_SNF_proposed_2026-06348.xml", # was 2026_SNF_proposed_2026-06348.xml - "2025_SNF_final_2025-2025.xml": "2025_SNF_final_2025-16907.xml", # was 2025_SNF_final_2025-16907.xml - "2024_SNF_final_2024-2024.xml": "2024_SNF_final_2024-22050.xml", # was 2024_SNF_final_2024-22050.xml -} - -def fix_filenames(): - """Fix filenames to keep doc ID unchanged but correct the year.""" - project_root = Path(__file__).parent.parent - data_dir = project_root / "data" - - print("Fixing filenames to keep doc ID unchanged...") - - for wrong_name, correct_name in CORRECT_NAMES.items(): - category = correct_name.split('_')[1] - wrong_path = data_dir / category / wrong_name - correct_path = data_dir / category / correct_name - - if wrong_path.exists(): - print(f" Fixing: {wrong_name} -> {correct_name}") - shutil.move(str(wrong_path), str(correct_path)) - else: - print(f" Warning: {wrong_name} not found") - -if __name__ == "__main__": - fix_filenames() \ No newline at end of file diff --git a/scripts/fix_processed_files.py b/scripts/fix_processed_files.py deleted file mode 100644 index e6e4d0ca..00000000 --- a/scripts/fix_processed_files.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix processed_files.json to match the renamed XML files. - -This script updates the file paths in processed_files.json to match -the corrected year naming convention. -""" - -import os -import json -import shutil -from pathlib import Path -import re - -# Year mapping: old_year -> new_year -YEAR_MAPPING = { - "2023": "2024", - "2024": "2025", - "2025": "2026" -} - -def get_new_file_path(old_file_path: str) -> str: - """ - Convert old file path to new file path with corrected year. - - Args: - old_file_path: Original file path (e.g., "MPFS/2023_MPFS_final_2023-24184.xml") - - Returns: - New file path with corrected year (e.g., "MPFS/2024_MPFS_final_2024-24184.xml") - """ - # Extract components from path: CATEGORY/YYYY_TYPE_TYPE_YYYY-XXXXX.xml - pattern = r'^([A-Z]+)/(\d{4})_([A-Z]+)_([a-z]+)_\d{4}-\d+\.xml$' - match = re.match(pattern, old_file_path) - - if not match: - print(f"Warning: Could not parse file path pattern: {old_file_path}") - return old_file_path - - category = match.group(1) - old_year = match.group(2) - file_type = match.group(3) - rule_type = match.group(4) - - # Get new year from mapping - if old_year in YEAR_MAPPING: - new_year = YEAR_MAPPING[old_year] - # Create new file path with corrected year - new_file_path = f"{category}/{new_year}_{file_type}_{rule_type}_{new_year}-{old_file_path.split('-')[1]}" - return new_file_path - else: - print(f"Warning: No mapping found for year {old_year} in {old_file_path}") - return old_file_path - -def main(): - """Main function to fix processed_files.json.""" - print("Fixing processed_files.json...") - print("Year mapping:") - for old_year, new_year in YEAR_MAPPING.items(): - print(f" {old_year} -> {new_year}") - - # Get project root directory - project_root = Path(__file__).parent.parent - processed_files_path = project_root / "rag_data" / "processed_files.json" - - if not os.path.exists(processed_files_path): - print(f"Error: {processed_files_path} not found") - return - - print(f"\nProcessing: {processed_files_path}") - - # Load current processed files - with open(processed_files_path, 'r') as f: - processed_files = json.load(f) - - print(f"Found {len(processed_files)} processed files") - - # Create backup - backup_path = str(processed_files_path) + '.backup2' - shutil.copy2(processed_files_path, backup_path) - print(f"Backup created: {backup_path}") - - # Update file paths - updated_files = {} - changes_made = 0 - - for old_path, file_info in processed_files.items(): - new_path = get_new_file_path(old_path) - - if old_path != new_path: - print(f" Updating: {old_path} -> {new_path}") - updated_files[new_path] = file_info - changes_made += 1 - else: - updated_files[old_path] = file_info - - if changes_made > 0: - # Write updated file - with open(processed_files_path, 'w') as f: - json.dump(updated_files, f, indent=2) - print(f"\nUpdated {changes_made} file paths") - else: - print("\nNo changes needed") - - print("\nVerification:") - # Check if all files exist - missing_files = [] - for file_path in updated_files.keys(): - full_path = project_root / "data" / file_path - if not os.path.exists(full_path): - missing_files.append(file_path) - print(f" ✗ {file_path} (file not found)") - else: - print(f" ✓ {file_path}") - - if missing_files: - print(f"\nWarning: {len(missing_files)} files are missing") - else: - print("\n✅ All files found!") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/fix_year_from_content.py b/scripts/fix_year_from_content.py deleted file mode 100644 index 6505ec80..00000000 --- a/scripts/fix_year_from_content.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix year naming by extracting the correct year from XML content. - -This script: -1. Extracts the target year from XML content (SUBJECT field) using regex (CY XXXX) -2. Renames files by replacing the year in filename but keeping doc ID unchanged -3. Updates processed_files.json, chunks.json, and faiss_metadata.json accordingly -""" - -import os -import json -import shutil -import re -import xml.etree.ElementTree as ET -from pathlib import Path -from typing import Dict, Optional - -def extract_year_from_xml(xml_path: str, category: str) -> Optional[str]: - """ - Extract target year from XML content using category-specific patterns. - - Args: - xml_path: Path to XML file - category: One of 'MPFS', 'HOSPICE', 'SNF' - Returns: - Extracted year (e.g., "2024") or None if not found - """ - try: - with open(xml_path, 'r', encoding='utf-8') as f: - content = f.read() - - if category == 'SNF': - # Federal Fiscal Year XXXX - match = re.search(r'Federal Fiscal Year\s*(\d{4})', content, re.IGNORECASE) - if match: - return match.group(1) - elif category == 'HOSPICE': - # FY XXXX - match = re.search(r'FY\s*(\d{4})', content, re.IGNORECASE) - if match: - return match.group(1) - elif category == 'MPFS': - # CY XXXX - match = re.search(r'CY\s*(\d{4})', content, re.IGNORECASE) - if match: - return match.group(1) - return None - except Exception as e: - print(f"Error reading {xml_path}: {e}") - return None - -def get_new_filename(old_filename: str, target_year: str) -> str: - """ - Generate new filename with corrected year but keeping doc ID unchanged. - - Args: - old_filename: Original filename (e.g., "2023_MPFS_final_2023-24184.xml") - target_year: Target year from XML content (e.g., "2024") - - Returns: - New filename (e.g., "2024_MPFS_final_2024-24184.xml") - """ - # Parse filename pattern: YYYY_TYPE_TYPE_YYYY-XXXXX.xml - pattern = r'^(\d{4})_([A-Z]+)_([a-z]+)_(\d{4})-(\d+)\.xml$' - match = re.match(pattern, old_filename) - - if not match: - print(f"Warning: Could not parse filename pattern: {old_filename}") - return old_filename - - old_year1 = match.group(1) # First year in filename - file_type = match.group(2) - rule_type = match.group(3) - old_year2 = match.group(4) # Second year in filename - doc_id = match.group(5) # Document ID - - # Create new filename with target year but keeping doc ID unchanged - new_filename = f"{target_year}_{file_type}_{rule_type}_{target_year}-{doc_id}.xml" - return new_filename - -def rename_xml_files_with_content_analysis(data_dir: str) -> Dict[str, str]: - """ - Rename XML files based on year extracted from content. - - Args: - data_dir: Path to data directory - - Returns: - Dictionary mapping old file paths to new file paths - """ - file_mapping = {} - - for category_dir in ['MPFS', 'HOSPICE', 'SNF']: - category_path = os.path.join(data_dir, category_dir) - if not os.path.exists(category_path): - continue - - print(f"\nProcessing {category_dir} directory...") - - for filename in os.listdir(category_path): - if filename.endswith('.xml'): - old_path = os.path.join(category_path, filename) - - # Extract target year from XML content - target_year = extract_year_from_xml(old_path, category_dir) - - if target_year is None: - print(f" Warning: Could not extract year from {filename} for {category_dir}, skipping") - continue - - # Generate new filename - new_filename = get_new_filename(filename, target_year) - new_path = os.path.join(category_path, new_filename) - - if old_path != new_path: - print(f" Renaming: {filename} -> {new_filename} (target year: {target_year})") - try: - shutil.move(old_path, new_path) - file_mapping[old_path] = new_path - except Exception as e: - print(f" Error renaming {filename}: {e}") - else: - print(f" No change needed: {filename} (already correct year: {target_year})") - - return file_mapping - -def update_processed_files(processed_files_path: str, file_mapping: Dict[str, str]): - """ - Update processed_files.json with new file paths. - - Args: - processed_files_path: Path to processed_files.json - file_mapping: Mapping of old to new file paths - """ - if not os.path.exists(processed_files_path): - print(f"Warning: {processed_files_path} not found") - return - - print(f"\nUpdating {processed_files_path}...") - - with open(processed_files_path, 'r') as f: - processed_files = json.load(f) - - updated_files = {} - changes_made = 0 - - for old_path, file_info in processed_files.items(): - # Convert relative path to absolute for comparison - old_abs_path = os.path.abspath(old_path) - - # Find corresponding new path - new_path = None - for old_file_path, new_file_path in file_mapping.items(): - if old_file_path == old_abs_path: - # Convert back to relative path - new_path = os.path.relpath(new_file_path, os.path.dirname(processed_files_path)) - break - - if new_path: - print(f" Updating: {old_path} -> {new_path}") - updated_files[new_path] = file_info - changes_made += 1 - else: - updated_files[old_path] = file_info - - if changes_made > 0: - # Backup original file - backup_path = processed_files_path + '.backup3' - shutil.copy2(processed_files_path, backup_path) - print(f" Backup created: {backup_path}") - - # Write updated file - with open(processed_files_path, 'w') as f: - json.dump(updated_files, f, indent=2) - print(f" Updated {changes_made} entries") - else: - print(" No changes needed") - -def update_chunks_file(chunks_path: str, file_mapping: Dict[str, str]): - """ - Update chunks.json with new file paths. - - Args: - chunks_path: Path to chunks.json - file_mapping: Mapping of old to new file paths - """ - if not os.path.exists(chunks_path): - print(f"Warning: {chunks_path} not found") - return - - print(f"\nUpdating {chunks_path}...") - - # This file might be large, so we'll process it line by line - backup_path = chunks_path + '.backup3' - shutil.copy2(chunks_path, backup_path) - print(f" Backup created: {backup_path}") - - changes_made = 0 - temp_path = chunks_path + '.tmp' - - with open(chunks_path, 'r') as infile, open(temp_path, 'w') as outfile: - for line in infile: - # Look for file paths in the JSON line - for old_file_path, new_file_path in file_mapping.items(): - old_relative_path = os.path.relpath(old_file_path, os.path.dirname(chunks_path)) - new_relative_path = os.path.relpath(new_file_path, os.path.dirname(chunks_path)) - - if old_relative_path in line: - line = line.replace(old_relative_path, new_relative_path) - changes_made += 1 - print(f" Updated chunk reference: {old_relative_path} -> {new_relative_path}") - - outfile.write(line) - - # Replace original with updated file - shutil.move(temp_path, chunks_path) - print(f" Updated {changes_made} chunk references") - -def update_faiss_metadata(faiss_metadata_path: str, file_mapping: Dict[str, str]): - """ - Update faiss_metadata.json with new file paths. - - Args: - faiss_metadata_path: Path to faiss_metadata.json - file_mapping: Mapping of old to new file paths - """ - if not os.path.exists(faiss_metadata_path): - print(f"Warning: {faiss_metadata_path} not found") - return - - print(f"\nUpdating {faiss_metadata_path}...") - - # This file might be large, so we'll process it line by line - backup_path = faiss_metadata_path + '.backup3' - shutil.copy2(faiss_metadata_path, backup_path) - print(f" Backup created: {backup_path}") - - changes_made = 0 - temp_path = faiss_metadata_path + '.tmp' - - with open(faiss_metadata_path, 'r') as infile, open(temp_path, 'w') as outfile: - for line in infile: - # Look for file paths in the JSON line - for old_file_path, new_file_path in file_mapping.items(): - old_relative_path = os.path.relpath(old_file_path, os.path.dirname(faiss_metadata_path)) - new_relative_path = os.path.relpath(new_file_path, os.path.dirname(faiss_metadata_path)) - - if old_relative_path in line: - line = line.replace(old_relative_path, new_relative_path) - changes_made += 1 - print(f" Updated metadata reference: {old_relative_path} -> {new_relative_path}") - - outfile.write(line) - - # Replace original with updated file - shutil.move(temp_path, faiss_metadata_path) - print(f" Updated {changes_made} metadata references") - -def main(): - """Main function to execute the year naming fix based on content analysis.""" - print("Starting year naming fix based on XML content analysis...") - - # Get project root directory - project_root = Path(__file__).parent.parent - data_dir = project_root / "data" - rag_data_dir = project_root / "rag_data" - - print(f"\nProject root: {project_root}") - print(f"Data directory: {data_dir}") - print(f"RAG data directory: {rag_data_dir}") - - # Step 1: Rename XML files based on content analysis - print("\n" + "="*60) - print("STEP 1: Analyzing XML content and renaming files") - print("="*60) - file_mapping = rename_xml_files_with_content_analysis(str(data_dir)) - - if not file_mapping: - print("No files were renamed. Exiting.") - return - - print(f"\nRenamed {len(file_mapping)} files:") - for old_path, new_path in file_mapping.items(): - print(f" {os.path.basename(old_path)} -> {os.path.basename(new_path)}") - - # Step 2: Update processed files - print("\n" + "="*60) - print("STEP 2: Updating processed files") - print("="*60) - processed_files_path = rag_data_dir / "processed_files.json" - update_processed_files(str(processed_files_path), file_mapping) - - # Step 3: Update chunks file - print("\n" + "="*60) - print("STEP 3: Updating chunks file") - print("="*60) - chunks_path = rag_data_dir / "chunks.json" - update_chunks_file(str(chunks_path), file_mapping) - - # Step 4: Update FAISS metadata - print("\n" + "="*60) - print("STEP 4: Updating FAISS metadata") - print("="*60) - faiss_metadata_path = rag_data_dir / "faiss_metadata.json" - update_faiss_metadata(str(faiss_metadata_path), file_mapping) - - print("\n" + "="*60) - print("Year naming fix completed!") - print("="*60) - print("\nSummary:") - print(f"- Renamed {len(file_mapping)} XML files based on content analysis") - print("- Updated processed_files.json") - print("- Updated chunks.json") - print("- Updated faiss_metadata.json") - print("\nBackup files created with .backup3 extension") - print("\nNext steps:") - print("1. Verify the changes look correct") - print("2. Test the search functionality") - print("3. If everything works, you can remove the .backup files") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/fix_year_naming.py b/scripts/fix_year_naming.py deleted file mode 100644 index 3092fa1d..00000000 --- a/scripts/fix_year_naming.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix year naming in XML files and corresponding processed results. - -This script corrects the year naming convention where: -- 2023 files should be 2024 (actual year 2024) -- 2024 files should be 2025 (actual year 2025) -- 2025 files should be 2026 (actual year 2026) - -The script will: -1. Rename XML files in data/ directories -2. Update processed_files.json with new file paths -3. Update chunks.json with new file paths -4. Update faiss_metadata.json with new file paths -""" - -import os -import json -import shutil -from pathlib import Path -from typing import Dict, List, Tuple -import re - -# Year mapping: old_year -> new_year -YEAR_MAPPING = { - "2023": "2024", - "2024": "2025", - "2025": "2026" -} - -def get_new_filename(old_filename: str) -> str: - """ - Convert old filename to new filename with corrected year. - - Args: - old_filename: Original filename (e.g., "2023_MPFS_final_2023-14624.xml") - - Returns: - New filename with corrected year (e.g., "2024_MPFS_final_2024-14624.xml") - """ - # Extract year from filename pattern: YYYY_TYPE_TYPE_YYYY-XXXXX.xml - pattern = r'^(\d{4})_([A-Z]+)_([a-z]+)_\d{4}-\d+\.xml$' - match = re.match(pattern, old_filename) - - if not match: - print(f"Warning: Could not parse filename pattern: {old_filename}") - return old_filename - - old_year = match.group(1) - file_type = match.group(2) - rule_type = match.group(3) - - # Get new year from mapping - if old_year in YEAR_MAPPING: - new_year = YEAR_MAPPING[old_year] - # Create new filename with corrected year - new_filename = f"{new_year}_{file_type}_{rule_type}_{new_year}-{old_filename.split('-')[1]}" - return new_filename - else: - print(f"Warning: No mapping found for year {old_year} in {old_filename}") - return old_filename - -def rename_xml_files(data_dir: str) -> Dict[str, str]: - """ - Rename XML files in data directory and return mapping of old to new paths. - - Args: - data_dir: Path to data directory - - Returns: - Dictionary mapping old file paths to new file paths - """ - file_mapping = {} - - for category_dir in ['MPFS', 'HOSPICE', 'SNF']: - category_path = os.path.join(data_dir, category_dir) - if not os.path.exists(category_path): - continue - - print(f"\nProcessing {category_dir} directory...") - - for filename in os.listdir(category_path): - if filename.endswith('.xml'): - old_path = os.path.join(category_path, filename) - new_filename = get_new_filename(filename) - new_path = os.path.join(category_path, new_filename) - - if old_path != new_path: - print(f" Renaming: {filename} -> {new_filename}") - try: - shutil.move(old_path, new_path) - file_mapping[old_path] = new_path - except Exception as e: - print(f" Error renaming {filename}: {e}") - else: - print(f" No change needed: {filename}") - - return file_mapping - -def update_processed_files(processed_files_path: str, file_mapping: Dict[str, str]): - """ - Update processed_files.json with new file paths. - - Args: - processed_files_path: Path to processed_files.json - file_mapping: Mapping of old to new file paths - """ - if not os.path.exists(processed_files_path): - print(f"Warning: {processed_files_path} not found") - return - - print(f"\nUpdating {processed_files_path}...") - - with open(processed_files_path, 'r') as f: - processed_files = json.load(f) - - updated_files = {} - changes_made = 0 - - for old_path, file_info in processed_files.items(): - # Convert relative path to absolute for comparison - old_abs_path = os.path.abspath(old_path) - - # Find corresponding new path - new_path = None - for old_file_path, new_file_path in file_mapping.items(): - if old_file_path == old_abs_path: - # Convert back to relative path - new_path = os.path.relpath(new_file_path, os.path.dirname(processed_files_path)) - break - - if new_path: - print(f" Updating: {old_path} -> {new_path}") - updated_files[new_path] = file_info - changes_made += 1 - else: - updated_files[old_path] = file_info - - if changes_made > 0: - # Backup original file - backup_path = processed_files_path + '.backup' - shutil.copy2(processed_files_path, backup_path) - print(f" Backup created: {backup_path}") - - # Write updated file - with open(processed_files_path, 'w') as f: - json.dump(updated_files, f, indent=2) - print(f" Updated {changes_made} entries") - else: - print(" No changes needed") - -def update_chunks_file(chunks_path: str, file_mapping: Dict[str, str]): - """ - Update chunks.json with new file paths. - - Args: - chunks_path: Path to chunks.json - file_mapping: Mapping of old to new file paths - """ - if not os.path.exists(chunks_path): - print(f"Warning: {chunks_path} not found") - return - - print(f"\nUpdating {chunks_path}...") - - # This file might be large, so we'll process it line by line - backup_path = chunks_path + '.backup' - shutil.copy2(chunks_path, backup_path) - print(f" Backup created: {backup_path}") - - changes_made = 0 - temp_path = chunks_path + '.tmp' - - with open(chunks_path, 'r') as infile, open(temp_path, 'w') as outfile: - for line in infile: - # Look for file paths in the JSON line - for old_file_path, new_file_path in file_mapping.items(): - old_relative_path = os.path.relpath(old_file_path, os.path.dirname(chunks_path)) - new_relative_path = os.path.relpath(new_file_path, os.path.dirname(chunks_path)) - - if old_relative_path in line: - line = line.replace(old_relative_path, new_relative_path) - changes_made += 1 - print(f" Updated chunk reference: {old_relative_path} -> {new_relative_path}") - - outfile.write(line) - - # Replace original with updated file - shutil.move(temp_path, chunks_path) - print(f" Updated {changes_made} chunk references") - -def update_faiss_metadata(faiss_metadata_path: str, file_mapping: Dict[str, str]): - """ - Update faiss_metadata.json with new file paths. - - Args: - faiss_metadata_path: Path to faiss_metadata.json - file_mapping: Mapping of old to new file paths - """ - if not os.path.exists(faiss_metadata_path): - print(f"Warning: {faiss_metadata_path} not found") - return - - print(f"\nUpdating {faiss_metadata_path}...") - - # This file might be large, so we'll process it line by line - backup_path = faiss_metadata_path + '.backup' - shutil.copy2(faiss_metadata_path, backup_path) - print(f" Backup created: {backup_path}") - - changes_made = 0 - temp_path = faiss_metadata_path + '.tmp' - - with open(faiss_metadata_path, 'r') as infile, open(temp_path, 'w') as outfile: - for line in infile: - # Look for file paths in the JSON line - for old_file_path, new_file_path in file_mapping.items(): - old_relative_path = os.path.relpath(old_file_path, os.path.dirname(faiss_metadata_path)) - new_relative_path = os.path.relpath(new_file_path, os.path.dirname(faiss_metadata_path)) - - if old_relative_path in line: - line = line.replace(old_relative_path, new_relative_path) - changes_made += 1 - print(f" Updated metadata reference: {old_relative_path} -> {new_relative_path}") - - outfile.write(line) - - # Replace original with updated file - shutil.move(temp_path, faiss_metadata_path) - print(f" Updated {changes_made} metadata references") - -def main(): - """Main function to execute the year naming fix.""" - print("Starting year naming fix...") - print("Year mapping:") - for old_year, new_year in YEAR_MAPPING.items(): - print(f" {old_year} -> {new_year}") - - # Get project root directory - project_root = Path(__file__).parent.parent - data_dir = project_root / "data" - rag_data_dir = project_root / "rag_data" - - print(f"\nProject root: {project_root}") - print(f"Data directory: {data_dir}") - print(f"RAG data directory: {rag_data_dir}") - - # Step 1: Rename XML files - print("\n" + "="*50) - print("STEP 1: Renaming XML files") - print("="*50) - file_mapping = rename_xml_files(str(data_dir)) - - if not file_mapping: - print("No files were renamed. Exiting.") - return - - print(f"\nRenamed {len(file_mapping)} files:") - for old_path, new_path in file_mapping.items(): - print(f" {os.path.basename(old_path)} -> {os.path.basename(new_path)}") - - # Step 2: Update processed files - print("\n" + "="*50) - print("STEP 2: Updating processed files") - print("="*50) - processed_files_path = rag_data_dir / "processed_files.json" - update_processed_files(str(processed_files_path), file_mapping) - - # Step 3: Update chunks file - print("\n" + "="*50) - print("STEP 3: Updating chunks file") - print("="*50) - chunks_path = rag_data_dir / "chunks.json" - update_chunks_file(str(chunks_path), file_mapping) - - # Step 4: Update FAISS metadata - print("\n" + "="*50) - print("STEP 4: Updating FAISS metadata") - print("="*50) - faiss_metadata_path = rag_data_dir / "faiss_metadata.json" - update_faiss_metadata(str(faiss_metadata_path), file_mapping) - - print("\n" + "="*50) - print("Year naming fix completed!") - print("="*50) - print("\nSummary:") - print(f"- Renamed {len(file_mapping)} XML files") - print("- Updated processed_files.json") - print("- Updated chunks.json") - print("- Updated faiss_metadata.json") - print("\nBackup files created with .backup extension") - print("\nNext steps:") - print("1. Verify the changes look correct") - print("2. Test the search functionality") - print("3. If everything works, you can remove the .backup files") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/quick_xml_label_check.py b/scripts/quick_xml_label_check.py deleted file mode 100644 index 1c6649c5..00000000 --- a/scripts/quick_xml_label_check.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -""" -quick_xml_label_check.py - -Quick script to check XML label consistency across data folders. -Provides a concise summary of whether different types use the same XML labels. - -Author: Fanxing Bu -Date: 2024-12-19 -""" - -import xml.etree.ElementTree as ET -from pathlib import Path -from typing import Dict, Set -import sys - -def extract_labels_from_file(xml_path: Path) -> Set[str]: - """Extract all XML labels from a single file.""" - try: - tree = ET.parse(xml_path) - root = tree.getroot() - return {elem.tag for elem in root.iter()} - except Exception as e: - print(f"Error processing {xml_path}: {e}") - return set() - -def analyze_folder_labels(folder_path: Path) -> Set[str]: - """Analyze all XML files in a folder and return all unique labels.""" - if not folder_path.exists(): - print(f"Folder {folder_path} does not exist") - return set() - - xml_files = list(folder_path.glob("*.xml")) - if not xml_files: - print(f"No XML files found in {folder_path}") - return set() - - all_labels = set() - for xml_file in xml_files: - labels = extract_labels_from_file(xml_file) - all_labels.update(labels) - - return all_labels - -def main(): - """Main function for quick XML label check.""" - data_dir = Path("data") - folder_types = ["HOSPICE", "MPFS", "SNF"] - - print("=" * 60) - print("QUICK XML LABEL CONSISTENCY CHECK") - print("=" * 60) - - # Analyze each folder - folder_labels = {} - for folder_type in folder_types: - folder_path = data_dir / folder_type - print(f"\nAnalyzing {folder_type}...") - - labels = analyze_folder_labels(folder_path) - folder_labels[folder_type] = labels - print(f" Found {len(labels)} unique XML labels") - - # Compare labels - print("\n" + "=" * 60) - print("COMPARISON RESULTS:") - print("=" * 60) - - if not folder_labels: - print("No data to compare") - return - - # Find common labels across all folders - common_labels = set.intersection(*folder_labels.values()) - print(f"\nCommon labels across all folders: {len(common_labels)}") - - # Check if all folders have the same label set - all_labels_union = set.union(*folder_labels.values()) - is_consistent = len(common_labels) == len(all_labels_union) - - print(f"All folders use same label set: {'✅ YES' if is_consistent else '❌ NO'}") - - if not is_consistent: - print("\nDifferences found:") - for folder_type, labels in folder_labels.items(): - other_labels = set.union(*[l for f, l in folder_labels.items() if f != folder_type]) - unique_labels = labels - other_labels - if unique_labels: - print(f" {folder_type} unique labels: {sorted(unique_labels)}") - - # Show label counts - print(f"\nLabel counts per folder:") - for folder_type, labels in folder_labels.items(): - print(f" {folder_type}: {len(labels)} labels") - - # Show common labels - if common_labels: - print(f"\nCommon labels ({len(common_labels)}):") - for label in sorted(common_labels): - print(f" - {label}") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/restore_and_fix.py b/scripts/restore_and_fix.py deleted file mode 100644 index eb563798..00000000 --- a/scripts/restore_and_fix.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -""" -Restore original filenames and then rename with correct logic. -""" - -import os -import shutil -from pathlib import Path - -# Original filename mapping (before the wrong rename) -ORIGINAL_NAMES = { - # MPFS - "2025_MPFS_final_2025-2025.xml": "2026_MPFS_final_2026-08676.xml", - "2024_MPFS_final_2024-2024.xml": "2025_MPFS_final_2025-02705.xml", - "2025_MPFS_proposed_2025-2025.xml": "2025_MPFS_proposed_2025-14828.xml", - "2024_MPFS_final_2024-2024.xml": "2024_MPFS_final_2024-24184.xml", - "2025_MPFS_final_2025-2025.xml": "2025_MPFS_final_2025-25382.xml", - "2024_MPFS_proposed_2024-2024.xml": "2024_MPFS_proposed_2024-14624.xml", - - # HOSPICE - "2024_HOSPICE_final_2024-2024.xml": "2024_HOSPICE_final_2024-16116.xml", - "2025_HOSPICE_final_2025-2025.xml": "2025_HOSPICE_final_2025-16910.xml", - "2025_HOSPICE_final_2025-2025.xml": "2025_HOSPICE_final_2025-22495.xml", - "2025_HOSPICE_proposed_2025-2025.xml": "2025_HOSPICE_proposed_2025-06921.xml", - "2026_HOSPICE_proposed_2026-2026.xml": "2026_HOSPICE_proposed_2026-06317.xml", - - # SNF - "2024_SNF_final_2024-2024.xml": "2025_SNF_final_2025-07522.xml", - "2025_SNF_proposed_2025-2025.xml": "2025_SNF_proposed_2025-06812.xml", - "2025_SNF_final_2025-2025.xml": "2025_SNF_final_2025-22504.xml", - "2024_SNF_final_2024-2024.xml": "2024_SNF_final_2024-16249.xml", - "2026_SNF_proposed_2026-2026.xml": "2026_SNF_proposed_2026-06348.xml", - "2025_SNF_final_2025-2025.xml": "2025_SNF_final_2025-16907.xml", - "2024_SNF_final_2024-2024.xml": "2024_SNF_final_2024-22050.xml", -} - -def restore_original_names(): - """Restore original filenames.""" - project_root = Path(__file__).parent.parent - data_dir = project_root / "data" - - print("Restoring original filenames...") - - for wrong_name, original_name in ORIGINAL_NAMES.items(): - category = original_name.split('_')[1] - wrong_path = data_dir / category / wrong_name - original_path = data_dir / category / original_name - - if wrong_path.exists(): - print(f" Restoring: {wrong_name} -> {original_name}") - shutil.move(str(wrong_path), str(original_path)) - else: - print(f" Warning: {wrong_name} not found") - -if __name__ == "__main__": - restore_original_names() \ No newline at end of file diff --git a/scripts/test_xml_auto_headings_analysis.py b/scripts/test_xml_auto_headings_analysis.py deleted file mode 100644 index 65635c45..00000000 --- a/scripts/test_xml_auto_headings_analysis.py +++ /dev/null @@ -1,38 +0,0 @@ -import pytest -from lxml import etree -from xml_auto_headings_analysis import get_text, is_heading, count_tokens, build_section_tree - -def test_get_text(): - elem = etree.Element('p') - elem.text = 'This is a test of the get_text function.' - assert get_text(elem, 3) == 'This is a ...' - assert get_text(elem, 10) == 'This is a test of the get_text function.' - assert get_text(None) == '' - elem2 = etree.Element('p') - assert get_text(elem2) == '' - -def test_is_heading(): - assert is_heading('CHAPTER I') - assert is_heading('1. Introduction') - assert is_heading('Section 2.1') - assert not is_heading('This is a normal sentence.') - assert is_heading('Short Title') - assert not is_heading('') - -def test_count_tokens(): - text = 'This is a test.' - n = count_tokens(text) - assert n > 0 - assert isinstance(n, int) - assert count_tokens('') == 0 - -def test_build_section_tree(): - xml = '''TitlePrefaceAmendmentAnother''' - root = etree.fromstring(xml) - tree = build_section_tree(root) - assert isinstance(tree, list) - assert len(tree) == 4 # 2 FP, 1 HD, 1 AMDPAR - tags = [n['tag'] for n in tree] - assert set(tags) == {'HD', 'FP', 'AMDPAR', 'FP'} - for node in tree: - assert 'tag' in node and 'text' in node and 'children' in node \ No newline at end of file diff --git a/scripts/test_xml_label_analyzer.py b/scripts/test_xml_label_analyzer.py deleted file mode 100644 index e116c6f1..00000000 --- a/scripts/test_xml_label_analyzer.py +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env python3 -""" -test_xml_label_analyzer.py - -Unit tests for the XML label analyzer functionality. - -Author: Fanxing Bu -Date: 2024-12-19 -""" - -import unittest -import tempfile -import os -from pathlib import Path -import xml.etree.ElementTree as ET -from analyze_xml_labels import XMLLabelAnalyzer - -class TestXMLLabelAnalyzer(unittest.TestCase): - """Test cases for XMLLabelAnalyzer class.""" - - def setUp(self): - """Set up test fixtures.""" - self.temp_dir = tempfile.mkdtemp() - self.test_data_dir = Path(self.temp_dir) / "test_data" - self.test_data_dir.mkdir() - - # Create test XML files - self.create_test_xml_files() - - def tearDown(self): - """Clean up test fixtures.""" - import shutil - shutil.rmtree(self.temp_dir) - - def create_test_xml_files(self): - """Create test XML files with known structures.""" - # Create HOSPICE folder - hospice_dir = self.test_data_dir / "HOSPICE" - hospice_dir.mkdir() - - # Test file 1 - basic structure - hospice_xml1 = """ - - - Test Agency - Test Subject -

Test paragraph

- - - Test Header -

More content

- -""" - - with open(hospice_dir / "test_hospice1.xml", "w") as f: - f.write(hospice_xml1) - - # Test file 2 - different structure - hospice_xml2 = """ - - - Test Agency - Test Subject - Footnote content - - - Test Header -
Test Section
- -""" - - with open(hospice_dir / "test_hospice2.xml", "w") as f: - f.write(hospice_xml2) - - # Create MPFS folder with different labels - mpfs_dir = self.test_data_dir / "MPFS" - mpfs_dir.mkdir() - - mpfs_xml = """ - - - Test Agency - Test Subject -

Test paragraph

- Test Appendix - - - Test Header - Table of Contents - -""" - - with open(mpfs_dir / "test_mpfs.xml", "w") as f: - f.write(mpfs_xml) - - # Create SNF folder - snf_dir = self.test_data_dir / "SNF" - snf_dir.mkdir() - - snf_xml = """ - - - Test Agency - Test Subject -

Test paragraph

- - - Test Header -

More content

- -""" - - with open(snf_dir / "test_snf.xml", "w") as f: - f.write(snf_xml) - - def test_extract_xml_labels(self): - """Test extracting XML labels from a single file.""" - analyzer = XMLLabelAnalyzer(str(self.test_data_dir)) - test_file = self.test_data_dir / "HOSPICE" / "test_hospice1.xml" - - labels = analyzer.extract_xml_labels(test_file) - expected_labels = {"RULE", "PREAMB", "AGENCY", "SUBJECT", "P", "SUPLINF", "HD"} - - self.assertEqual(labels, expected_labels) - - def test_analyze_folder(self): - """Test analyzing a folder of XML files.""" - analyzer = XMLLabelAnalyzer(str(self.test_data_dir)) - result = analyzer.analyze_folder("HOSPICE") - - self.assertEqual(result["folder_name"], "HOSPICE") - self.assertEqual(result["file_count"], 2) - self.assertIn("test_hospice1.xml", result["file_labels"]) - self.assertIn("test_hospice2.xml", result["file_labels"]) - - # Check that all expected labels are found - all_labels = result["all_labels"] - expected_labels = {"RULE", "PREAMB", "AGENCY", "SUBJECT", "P", "SUPLINF", "HD", "FTNT", "SECTION"} - self.assertEqual(all_labels, expected_labels) - - def test_compare_folders(self): - """Test comparing labels across folders.""" - analyzer = XMLLabelAnalyzer(str(self.test_data_dir)) - results = analyzer.compare_folders() - - # Check that all folders were analyzed - self.assertIn("HOSPICE", results["folder_results"]) - self.assertIn("MPFS", results["folder_results"]) - self.assertIn("SNF", results["folder_results"]) - - # Check that MPFS has unique labels - unique_per_folder = results["unique_per_folder"] - self.assertIn("MPFS", unique_per_folder) - mpfs_unique = unique_per_folder["MPFS"] - self.assertIn("APPENDIX", mpfs_unique) - self.assertIn("CONTENTS", mpfs_unique) - - # Check consistency - consistency = results["consistency_check"] - self.assertFalse(consistency["label_sets_are_equal"]) # Should be different due to MPFS unique labels - - def test_common_labels(self): - """Test finding common labels across folders.""" - analyzer = XMLLabelAnalyzer(str(self.test_data_dir)) - results = analyzer.compare_folders() - - common_labels = results["common_across_folders"] - expected_common = {"RULE", "PREAMB", "AGENCY", "SUBJECT", "P", "SUPLINF", "HD"} - - self.assertEqual(common_labels, expected_common) - - def test_empty_folder(self): - """Test handling of empty folder.""" - analyzer = XMLLabelAnalyzer(str(self.test_data_dir)) - - # Create empty folder - empty_dir = self.test_data_dir / "EMPTY" - empty_dir.mkdir() - - result = analyzer.analyze_folder("EMPTY") - self.assertEqual(result, {}) # Should return empty dict for empty folder - - def test_nonexistent_folder(self): - """Test handling of nonexistent folder.""" - analyzer = XMLLabelAnalyzer(str(self.test_data_dir)) - result = analyzer.analyze_folder("NONEXISTENT") - self.assertEqual(result, {}) # Should return empty dict for nonexistent folder - -def run_tests(): - """Run all tests.""" - unittest.main(verbosity=2) - -if __name__ == "__main__": - run_tests() \ No newline at end of file diff --git a/scripts/test_year_extraction.py b/scripts/test_year_extraction.py deleted file mode 100644 index 3d615f3d..00000000 --- a/scripts/test_year_extraction.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script to verify year extraction from XML content. -""" - -import os -import re -from pathlib import Path - -def extract_year_from_xml(xml_path: str, category: str): - """Extract target year from XML content using category-specific patterns.""" - try: - with open(xml_path, 'r', encoding='utf-8') as f: - content = f.read() - if category == 'SNF': - match = re.search(r'Federal Fiscal Year\s*(\d{4})', content, re.IGNORECASE) - if match: - return match.group(1), 'Federal Fiscal Year' - elif category == 'HOSPICE': - match = re.search(r'FY\s*(\d{4})', content, re.IGNORECASE) - if match: - return match.group(1), 'FY' - elif category == 'MPFS': - match = re.search(r'CY\s*(\d{4})', content, re.IGNORECASE) - if match: - return match.group(1), 'CY' - return None, None - except Exception as e: - print(f"Error reading {xml_path}: {e}") - return None, None - -def main(): - """Test year extraction on all XML files by category.""" - print("Testing year extraction from XML content by category...") - - # Get project root directory - project_root = Path(__file__).parent.parent - data_dir = project_root / "data" - - print(f"Data directory: {data_dir}") - - for category_dir in ['MPFS', 'HOSPICE', 'SNF']: - category_path = data_dir / category_dir - if not category_path.exists(): - continue - - print(f"\n{category_dir}:") - - for filename in os.listdir(category_path): - if filename.endswith('.xml'): - xml_path = category_path / filename - extracted_year, source = extract_year_from_xml(str(xml_path), category_dir) - - if extracted_year: - print(f" ✓ {filename} -> {source} {extracted_year}") - else: - print(f" ✗ {filename} -> No year found") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/test_year_fix.py b/scripts/test_year_fix.py deleted file mode 100644 index 6ee3ee9b..00000000 --- a/scripts/test_year_fix.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script to verify the year naming fix. - -This script checks: -1. XML file naming consistency -2. Processed files mapping -3. Chunks file references -4. FAISS metadata references -""" - -import os -import json -from pathlib import Path -import re - -def check_xml_files(data_dir: str): - """Check XML file naming consistency.""" - print("Checking XML file naming...") - - issues = [] - for category_dir in ['MPFS', 'HOSPICE', 'SNF']: - category_path = os.path.join(data_dir, category_dir) - if not os.path.exists(category_path): - continue - - print(f"\n {category_dir}:") - for filename in os.listdir(category_path): - if filename.endswith('.xml'): - # Check if filename follows expected pattern - pattern = r'^(\d{4})_([A-Z]+)_([a-z]+)_\d{4}-\d+\.xml$' - match = re.match(pattern, filename) - - if match: - year = match.group(1) - file_type = match.group(2) - rule_type = match.group(3) - - # Check if year is in expected range (2024-2026) - if year in ['2024', '2025', '2026']: - print(f" ✓ {filename}") - else: - print(f" ✗ {filename} (unexpected year: {year})") - issues.append(f"Unexpected year {year} in {filename}") - else: - print(f" ✗ {filename} (doesn't match pattern)") - issues.append(f"Pattern mismatch in {filename}") - - return issues - -def check_processed_files(processed_files_path: str): - """Check processed_files.json consistency.""" - print("\nChecking processed_files.json...") - - if not os.path.exists(processed_files_path): - print(" ✗ File not found") - return ["processed_files.json not found"] - - issues = [] - with open(processed_files_path, 'r') as f: - processed_files = json.load(f) - - print(f" Found {len(processed_files)} processed files") - - # Get project root directory for file existence check - project_root = Path(processed_files_path).parent.parent - - for file_path, file_info in processed_files.items(): - # Check if file exists in data directory - full_path = project_root / "data" / file_path - if not os.path.exists(full_path): - print(f" ✗ {file_path} (file not found)") - issues.append(f"File not found: {file_path}") - else: - print(f" ✓ {file_path}") - - # Check if path follows expected pattern - if not file_path.endswith('.xml'): - print(f" ✗ {file_path} (not an XML file)") - issues.append(f"Not an XML file: {file_path}") - - return issues - -def check_chunks_file(chunks_path: str): - """Check chunks.json file references.""" - print("\nChecking chunks.json...") - - if not os.path.exists(chunks_path): - print(" ✗ File not found") - return ["chunks.json not found"] - - issues = [] - chunk_count = 0 - file_references = set() - - with open(chunks_path, 'r') as f: - for line in f: - chunk_count += 1 - # Look for file paths in the line - if '"file_path"' in line: - # Extract file path from JSON - match = re.search(r'"file_path":\s*"([^"]+)"', line) - if match: - file_path = match.group(1) - file_references.add(file_path) - - print(f" Found {chunk_count} chunks") - print(f" Found {len(file_references)} unique file references") - - for file_path in sorted(file_references): - if not os.path.exists(file_path): - print(f" ✗ {file_path} (file not found)") - issues.append(f"File not found in chunks: {file_path}") - else: - print(f" ✓ {file_path}") - - return issues - -def check_faiss_metadata(faiss_metadata_path: str): - """Check faiss_metadata.json file references.""" - print("\nChecking faiss_metadata.json...") - - if not os.path.exists(faiss_metadata_path): - print(" ✗ File not found") - return ["faiss_metadata.json not found"] - - issues = [] - metadata_count = 0 - file_references = set() - - with open(faiss_metadata_path, 'r') as f: - for line in f: - metadata_count += 1 - # Look for file paths in the line - if '"file_path"' in line: - # Extract file path from JSON - match = re.search(r'"file_path":\s*"([^"]+)"', line) - if match: - file_path = match.group(1) - file_references.add(file_path) - - print(f" Found {metadata_count} metadata entries") - print(f" Found {len(file_references)} unique file references") - - for file_path in sorted(file_references): - if not os.path.exists(file_path): - print(f" ✗ {file_path} (file not found)") - issues.append(f"File not found in metadata: {file_path}") - else: - print(f" ✓ {file_path}") - - return issues - -def main(): - """Main test function.""" - print("="*60) - print("YEAR NAMING FIX VERIFICATION") - print("="*60) - - # Get project root directory - project_root = Path(__file__).parent.parent - data_dir = project_root / "data" - rag_data_dir = project_root / "rag_data" - - print(f"Project root: {project_root}") - print(f"Data directory: {data_dir}") - print(f"RAG data directory: {rag_data_dir}") - - all_issues = [] - - # Check XML files - xml_issues = check_xml_files(str(data_dir)) - all_issues.extend(xml_issues) - - # Check processed files - processed_files_path = rag_data_dir / "processed_files.json" - processed_issues = check_processed_files(str(processed_files_path)) - all_issues.extend(processed_issues) - - # Check chunks file - chunks_path = rag_data_dir / "chunks.json" - chunks_issues = check_chunks_file(str(chunks_path)) - all_issues.extend(chunks_issues) - - # Check FAISS metadata - faiss_metadata_path = rag_data_dir / "faiss_metadata.json" - metadata_issues = check_faiss_metadata(str(faiss_metadata_path)) - all_issues.extend(metadata_issues) - - # Summary - print("\n" + "="*60) - print("VERIFICATION SUMMARY") - print("="*60) - - if all_issues: - print(f"❌ Found {len(all_issues)} issues:") - for issue in all_issues: - print(f" - {issue}") - print("\nPlease fix these issues before proceeding.") - else: - print("✅ All checks passed! The year naming fix appears to be successful.") - print("\nNext steps:") - print("1. Test the search functionality") - print("2. Verify search results are correct") - print("3. If everything works, you can remove the .backup files") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/xml_auto_headings_analysis.py b/scripts/xml_auto_headings_analysis.py deleted file mode 100644 index 533e0884..00000000 --- a/scripts/xml_auto_headings_analysis.py +++ /dev/null @@ -1,234 +0,0 @@ -""" -xml_auto_headings_analysis.py - -Script to analyze XML tags, print tag frequencies, sample contents, auto-infer likely title, heading, and paragraph tags, count total words/tokens, and build a section tree for structural tags (e.g., HD, FP, AMDPAR). - -Usage: - python scripts/xml_auto_headings_analysis.py data/2025-06008.xml - -Data format for section tree: - { - 'tag': str, # e.g., 'HD' - 'text': str, # text content - 'children': [ ... ] # list of child nodes (same structure) - } -""" -import sys -from lxml import etree -from collections import Counter, defaultdict -import re - -try: - import tiktoken -except ImportError: - tiktoken = None - print("Warning: tiktoken not installed, token count will be word count.") - -SECTION_TAGS = {'HD', 'FP', 'AMDPAR'} # 可根据实际需要扩展 - -# Tag priority levels for section tree construction -TAG_LEVELS = { - 'RULE': 0, - 'PREAMB': 1, 'SUPLINF': 1, 'REGTEXT': 1, - 'HD': 2, - 'FP': 3, 'AMDPAR': 3, 'SECTION': 3, 'SUBPART': 3, - 'P': 4, 'E': 4, 'FTNT': 4, 'NOTE': 4, - 'PRTPAGE': 5, 'GID': 5, 'BILCOD': 5, 'AUTH': 5 -} - -# Helper to get tag level, default to 99 for unknown tags -get_tag_level = lambda tag: TAG_LEVELS.get(tag, 99) - -def get_text(elem, max_words=8): - """ - Get a short preview of element text. - """ - if elem is None or elem.text is None: - return '' - words = elem.text.strip().split() - if len(words) > max_words: - return ' '.join(words[:max_words]) + ' ...' - return ' '.join(words) - -def is_heading(text): - """ - Heuristic to determine if text looks like a heading. - """ - if not text: - return False - if text.isupper() and len(text.split()) <= 10: - return True - if re.match(r'^[A-Z]?[0-9]+[\.-]', text): - return True - if len(text.split()) <= 8 and text.istitle(): - return True - return False - -def count_tokens(text, encoding_name='cl100k_base'): - """ - Count tokens using tiktoken if available, else fallback to word count. - >>> count_tokens('This is a test.') > 0 - True - """ - if not text: - return 0 - if tiktoken: - enc = tiktoken.get_encoding(encoding_name) - return len(enc.encode(text)) - else: - return len(text.strip().split()) - -def build_section_tree(elem, parent_level=-1): - """ - Build a nested section tree according to tag priority levels. - Returns a list of section nodes. - Each node: {'tag': str, 'text': str, 'children': list} - """ - nodes = [] - for child in elem: - tag = child.tag - level = get_tag_level(tag) - node = { - 'tag': tag, - 'text': get_text(child, 20), - 'children': build_section_tree(child, level) - } - nodes.append(node) - return nodes - -def print_section_tree(nodes, level=0, max_words=5, max_per_tag=3): - """ - Pretty print the section tree, showing structure and a short content preview. - At each level, group by tag, show up to max_per_tag examples per tag, recursively. - Each node shows only the first max_words words of its text, with ellipsis if longer. - """ - tag_groups = defaultdict(list) - for node in nodes: - tag_groups[node['tag']].append(node) - for tag, group in tag_groups.items(): - for i, node in enumerate(group): - if i >= max_per_tag: - print(' ' * level + f"... ({len(group) - max_per_tag} more {tag})") - break - words = node['text'].split() - text = ' '.join(words[:max_words]) - if len(words) > max_words: - text += ' ...' - print(' ' * level + f"- {node['tag']}: {text}") - print_section_tree(node['children'], level+1, max_words, max_per_tag) - -def main(xml_path): - """ - Main analysis function. - Args: - xml_path (str): Path to XML file. - """ - try: - tree = etree.parse(xml_path) - except Exception as e: - print(f"Error parsing XML: {e}") - sys.exit(1) - root = tree.getroot() - - tag_counter = Counter() - tag_samples = defaultdict(list) - tag_lengths = defaultdict(list) - total_words = 0 - total_tokens = 0 - - # Traverse all elements, collect stats and count words/tokens - for elem in root.iter(): - tag_counter[elem.tag] += 1 - if elem.text and elem.text.strip(): - sample = get_text(elem, 12) - if len(tag_samples[elem.tag]) < 3: - tag_samples[elem.tag].append(sample) - word_count = len(elem.text.strip().split()) - tag_lengths[elem.tag].append(word_count) - total_words += word_count - total_tokens += count_tokens(elem.text) - - print('=== Tag Frequency & Sample Content ===') - for tag, count in tag_counter.most_common(): - print(f'- {tag}: {count}') - for i, sample in enumerate(tag_samples[tag]): - print(f' Sample {i+1}: {sample}') - print() - - # Print likely heuristics explanation - print('=== Heuristics for "Likely" Tags ===') - print('- Likely title: tag that appears only once and has no more than 15 words.') - print('- Likely heading: tag that appears 5-200 times, with short text that looks like a heading (all caps, numbered, or title case).') - print('- Likely paragraph: tag that appears more than 100 times and has at least one sample with more than 10 words.') - print() - - # Heuristic: likely title tag is the first tag with only one occurrence and short text - likely_title = None - for tag, count in tag_counter.items(): - if count == 1 and tag_samples[tag]: - text = tag_samples[tag][0] - if len(text.split()) <= 15: - likely_title = (tag, text) - break - print('=== Likely Article Title ===') - if likely_title: - print(f'Tag: {likely_title[0]} | Content: {likely_title[1]}') - else: - print('Not found') - print() - - # Heuristic: likely heading tags are those with moderate frequency, short text, and heading-like features - heading_candidates = [] - for tag, samples in tag_samples.items(): - if 5 <= tag_counter[tag] <= 200: - for s in samples: - if is_heading(s): - heading_candidates.append((tag, s)) - break - heading_tags = sorted(set([t for t, _ in heading_candidates])) - print('=== Likely Heading Tags ===') - for tag in heading_tags: - print(f'- {tag} (count: {tag_counter[tag]})') - for s in tag_samples[tag]: - if is_heading(s): - print(f' Example: {s}') - print() - - # Heuristic: likely paragraph tags are those with high frequency and long text - para_candidates = [] - for tag, lens in tag_lengths.items(): - if tag_counter[tag] > 100 and sum(l > 10 for l in lens) > 0: - para_candidates.append(tag) - print('=== Likely Paragraph Tags ===') - for tag in para_candidates: - print(f'- {tag} (count: {tag_counter[tag]})') - for s in tag_samples[tag]: - print(f' Example: {s}') - print() - - # Print summary - print('=== Summary ===') - print(f'Article title tag: {likely_title[0] if likely_title else "Not found"}') - print(f'Heading tags: {", ".join(heading_tags) if heading_tags else "Not found"}') - print(f'Paragraph tags: {", ".join(para_candidates) if para_candidates else "Not found"}') - print() - - # Print total word/token count - print('=== Word/Token Count ===') - print(f'Total words: {total_words}') - print(f'Total tokens: {total_tokens}') - print() - - # Build and print section tree (all tags) - print('=== Section Tree (all tags, grouped, up to 3 per tag per level) ===') - section_tree = build_section_tree(root) - if section_tree: - print_section_tree(section_tree) - else: - print('No section structure found.') - -if __name__ == '__main__': - if len(sys.argv) != 2: - print('Usage: python scripts/xml_auto_headings_analysis.py ') - sys.exit(1) - main(sys.argv[1]) \ No newline at end of file diff --git a/test_cases_data_fetcher_auto_update.md b/test_cases_data_fetcher_auto_update.md deleted file mode 100644 index 10ed7c98..00000000 --- a/test_cases_data_fetcher_auto_update.md +++ /dev/null @@ -1,1100 +0,0 @@ -# Test Cases for Data Fetcher and Auto Update Pipeline - -**Author:** Fanxing Bu -**Module:** `app/core/data_fetcher/fetch_regulations.py` and `app/core/auto_update_pipeline.py` - -## Overview - -This document provides comprehensive test cases for all functions in the data fetcher and auto update pipeline modules. Each test case includes: -- Function description -- Input/output specifications -- Test scenarios (normal, edge cases, error conditions) -- Sample data structures -- Expected behaviors - ---- - -## Module: `app/core/data_fetcher/fetch_regulations.py` - -### 1. `setup_logging(verbose: bool = False) -> logging.Logger` - -**Function Description:** Sets up logging configuration with specified verbosity level. - -**Test Cases:** - -#### TC-001: Default Logging Setup -- **Description:** Test logging setup with default parameters (verbose=False) -- **Input:** `verbose=False` -- **Expected Output:** Logger instance with INFO level -- **Test Steps:** - 1. Call `setup_logging(False)` - 2. Verify logger level is INFO - 3. Verify logger format includes timestamp, level, and message - -#### TC-002: Verbose Logging Setup -- **Description:** Test logging setup with verbose mode enabled -- **Input:** `verbose=True` -- **Expected Output:** Logger instance with DEBUG level -- **Test Steps:** - 1. Call `setup_logging(True)` - 2. Verify logger level is DEBUG - 3. Verify logger format is correct - -#### TC-003: Logger Functionality Test -- **Description:** Test that created logger can log messages -- **Input:** `verbose=True` -- **Expected Output:** Logger that can log debug, info, warning, error messages -- **Test Steps:** - 1. Create logger with `setup_logging(True)` - 2. Log messages at different levels - 3. Verify messages are properly formatted - ---- - -### 2. `get_single_document(doc_number: str) -> Optional[Dict]` - -**Function Description:** Fetches a single document by its document number from Federal Register API. - -**Test Cases:** - -#### TC-004: Valid Document Fetch -- **Description:** Test fetching a valid document number -- **Input:** `doc_number="2024-06431"` -- **Expected Output:** Dictionary with document data -- **Sample Response:** -```json -{ - "document_number": "2024-06431", - "title": "Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update", - "type": "Rule", - "publication_date": "2024-03-28", - "html_url": "https://www.federalregister.gov/documents/2024/03/28/2024-06431/...", - "pdf_url": "https://www.govinfo.gov/content/pkg/FR-2024-03-28/pdf/2024-06431.pdf", - "full_text_xml_url": "https://www.federalregister.gov/documents/full_text/xml/2024/2024-06431.xml" -} -``` - -#### TC-005: Invalid Document Number -- **Description:** Test fetching with invalid document number -- **Input:** `doc_number="invalid-doc-123"` -- **Expected Output:** `None` -- **Test Steps:** - 1. Call function with invalid document number - 2. Verify function returns None - 3. Verify error is logged - -#### TC-006: Empty Document Number -- **Description:** Test with empty document number -- **Input:** `doc_number=""` -- **Expected Output:** `None` -- **Test Steps:** - 1. Call function with empty string - 2. Verify function returns None - 3. Verify error is logged - -#### TC-007: Network Error Handling -- **Description:** Test behavior when network request fails -- **Input:** `doc_number="2024-06431"` (with mocked network failure) -- **Expected Output:** `None` -- **Test Steps:** - 1. Mock requests.get to raise RequestException - 2. Call function - 3. Verify function returns None - 4. Verify error is logged - ---- - -### 3. `get_latest_documents(days: int = 365) -> List[Dict]` - -**Function Description:** Fetches latest documents from Federal Register API with pagination support. - -**Test Cases:** - -#### TC-008: Default Days Parameter -- **Description:** Test with default days parameter (365) -- **Input:** `days=365` -- **Expected Output:** List of document dictionaries -- **Test Steps:** - 1. Call function with default parameter - 2. Verify returned list contains document dictionaries - 3. Verify documents have required fields (document_number, title, type, publication_date) - -#### TC-009: Custom Days Parameter -- **Description:** Test with custom days parameter -- **Input:** `days=30` -- **Expected Output:** List of documents from last 30 days -- **Test Steps:** - 1. Call function with days=30 - 2. Verify API call uses correct date range - 3. Verify returned documents are within date range - -#### TC-010: Pagination Handling -- **Description:** Test handling of multiple pages of results -- **Input:** `days=365` (with mocked multi-page response) -- **Expected Output:** Combined list from all pages -- **Test Steps:** - 1. Mock API to return multiple pages - 2. Call function - 3. Verify all pages are processed - 4. Verify delay between requests - -#### TC-011: Empty Results -- **Description:** Test when no documents are found -- **Input:** `days=1` (very recent) -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Call function with very recent date range - 2. Verify empty list is returned - 3. Verify no errors are logged - -#### TC-012: API Error Handling -- **Description:** Test behavior when API request fails -- **Input:** `days=365` (with mocked API failure) -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Mock requests.get to raise RequestException - 2. Call function - 3. Verify empty list is returned - 4. Verify error is logged - ---- - -### 4. `is_valid_xml(filepath: Path) -> bool` - -**Function Description:** Validates if a file is a valid XML file using lxml parser. - -**Test Cases:** - -#### TC-013: Valid XML File -- **Description:** Test with valid XML file -- **Input:** Path to valid XML file -- **Expected Output:** `True` -- **Sample XML:** -```xml - - - - 2024-06431 - 2024-03-28 - Rule - Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update - - -``` - -#### TC-014: Invalid XML File -- **Description:** Test with malformed XML file -- **Input:** Path to invalid XML file -- **Expected Output:** `False` -- **Sample Invalid XML:** -```xml - - - - 2024-06431 - 2024-03-28 - Rule - Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update - - -``` - -#### TC-015: Non-XML File -- **Description:** Test with non-XML file (e.g., text file) -- **Input:** Path to text file -- **Expected Output:** `False` -- **Sample Content:** -``` -This is a plain text file, not XML -``` - -#### TC-016: Non-existent File -- **Description:** Test with non-existent file path -- **Input:** Path to non-existent file -- **Expected Output:** `False` -- **Test Steps:** - 1. Call function with non-existent path - 2. Verify function returns False - 3. Verify no exception is raised - -#### TC-017: Empty XML File -- **Description:** Test with empty XML file -- **Input:** Path to empty file -- **Expected Output:** `False` -- **Test Steps:** - 1. Create empty file - 2. Call function - 3. Verify function returns False - ---- - -### 5. `extract_year_from_title(doc: Dict, program_type: str) -> Optional[str]` - -**Function Description:** Extracts year from document title using regex patterns for different program types. - -**Test Cases:** - -#### TC-018: MPFS Calendar Year Extraction -- **Description:** Test year extraction for MPFS documents -- **Input:** - - `doc={"title": "Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update"}` - - `program_type="MPFS"` -- **Expected Output:** `"2025"` -- **Test Patterns:** - - "CY 2025" → "2025" - - "Calendar Year (CY) 2025" → "2025" - - "Calendar Year 2025" → "2025" - -#### TC-019: HOSPICE Fiscal Year Extraction -- **Description:** Test year extraction for HOSPICE documents -- **Input:** - - `doc={"title": "Medicare Program; Fiscal Year (FY) 2025 Hospice Wage Index Update"}` - - `program_type="HOSPICE"` -- **Expected Output:** `"2025"` -- **Test Patterns:** - - "FY 2025" → "2025" - - "Fiscal Year (FY) 2025" → "2025" - - "Fiscal Year 2025" → "2025" - -#### TC-020: SNF Federal Fiscal Year Extraction -- **Description:** Test year extraction for SNF documents -- **Input:** - - `doc={"title": "Medicare Program; Federal Fiscal Year 2025 Prospective Payment System and Consolidated Billing for Skilled Nursing Facilities"}` - - `program_type="SNF"` -- **Expected Output:** `"2025"` -- **Test Pattern:** - - "Federal Fiscal Year 2025" → "2025" - -#### TC-021: No Year Found -- **Description:** Test when no year pattern is found in title -- **Input:** - - `doc={"title": "Medicare Program; General Update"}` - - `program_type="MPFS"` -- **Expected Output:** `None` - -#### TC-022: Case Insensitive Matching -- **Description:** Test case insensitive regex matching -- **Input:** - - `doc={"title": "Medicare Program; calendar year (cy) 2025 update"}` - - `program_type="MPFS"` -- **Expected Output:** `"2025"` - -#### TC-023: Multiple Year Patterns -- **Description:** Test when multiple year patterns exist (should return first match) -- **Input:** - - `doc={"title": "Medicare Program; Calendar Year (CY) 2025 and Fiscal Year 2026 Update"}` - - `program_type="MPFS"` -- **Expected Output:** `"2025"` - -#### TC-024: Invalid Program Type -- **Description:** Test with unsupported program type -- **Input:** - - `doc={"title": "Medicare Program; Calendar Year (CY) 2025 Update"}` - - `program_type="INVALID"` -- **Expected Output:** `None` - ---- - -### 6. `detect_program_type(doc: Dict) -> Tuple[bool, str]` - -**Function Description:** Detects program type from document title using keyword matching. - -**Test Cases:** - -#### TC-025: MPFS Detection -- **Description:** Test detection of MPFS program type -- **Input:** `doc={"title": "Medicare Physician Fee Schedule Update"}` -- **Expected Output:** `(True, "MPFS")` -- **Test Keywords:** - - "medicare physician fee schedule" - - "physician fee schedule" - - "mpfs" - -#### TC-026: HOSPICE Detection -- **Description:** Test detection of HOSPICE program type -- **Input:** `doc={"title": "Hospice Wage Index Update"}` -- **Expected Output:** `(True, "HOSPICE")` -- **Test Keywords:** - - "hospice wage" - - "hospice payment" - - "hospice quality" - -#### TC-027: SNF Detection -- **Description:** Test detection of SNF program type -- **Input:** `doc={"title": "Prospective Payment System and Consolidated Billing for Skilled Nursing Facilities"}` -- **Expected Output:** `(True, "SNF")` -- **Test Keywords:** - - "skilled nursing facility" - - "snf" - - "nursing facility" - - "consolidated billing" - -#### TC-028: Correction Document Detection -- **Description:** Test detection of correction documents (should be skipped) -- **Input:** `doc={"title": "Medicare Program; Correction to Physician Fee Schedule"}` -- **Expected Output:** `(False, "")` - -#### TC-029: Unrecognized Program Type -- **Description:** Test with unrecognized program type -- **Input:** `doc={"title": "Medicare Program; General Update"}` -- **Expected Output:** `(False, "")` - -#### TC-030: Case Insensitive Detection -- **Description:** Test case insensitive keyword matching -- **Input:** `doc={"title": "MEDICARE PHYSICIAN FEE SCHEDULE UPDATE"}` -- **Expected Output:** `(True, "MPFS")` - -#### TC-031: Empty Title -- **Description:** Test with empty title -- **Input:** `doc={"title": ""}` -- **Expected Output:** `(False, "")` - ---- - -### 7. `download_xml(doc: Dict, save_dir: Path, logger: Optional[logging.Logger] = None) -> bool` - -**Function Description:** Downloads XML file for a document and saves it with standardized naming. - -**Test Cases:** - -#### TC-032: Successful Download -- **Description:** Test successful XML download -- **Input:** - - `doc={"document_number": "2024-06431", "title": "Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update", "type": "Rule", "publication_date": "2024-03-28"}` - - `save_dir=Path("data")` -- **Expected Output:** `True` -- **Expected File:** `data/MPFS/2025_MPFS_final_2024-06431.xml` - -#### TC-033: File Already Exists -- **Description:** Test when file already exists and is valid -- **Input:** Existing valid XML file -- **Expected Output:** `True` -- **Test Steps:** - 1. Create valid XML file at expected location - 2. Call download function - 3. Verify function returns True without downloading - -#### TC-034: Invalid XML Download -- **Description:** Test when downloaded file is not valid XML -- **Input:** Document that returns invalid XML -- **Expected Output:** `False` -- **Test Steps:** - 1. Mock response to return invalid XML - 2. Call download function - 3. Verify function returns False - 4. Verify downloaded file is deleted - -#### TC-035: Missing Document Information -- **Description:** Test with missing required document fields -- **Input:** `doc={"title": "Test Document"}` (missing document_number, publication_date, type) -- **Expected Output:** `False` - -#### TC-036: Unrecognized Program Type -- **Description:** Test with document that has unrecognized program type -- **Input:** `doc={"document_number": "2024-06431", "title": "General Update", "type": "Rule", "publication_date": "2024-03-28"}` -- **Expected Output:** `False` - -#### TC-037: Year Extraction Failure -- **Description:** Test when year cannot be extracted from title -- **Input:** `doc={"document_number": "2024-06431", "title": "Medicare Program Update", "type": "Rule", "publication_date": "2024-03-28"}` -- **Expected Output:** `False` - -#### TC-038: Network Error During Download -- **Description:** Test behavior when network request fails -- **Input:** Valid document (with mocked network failure) -- **Expected Output:** `False` -- **Test Steps:** - 1. Mock requests.get to raise RequestException - 2. Call download function - 3. Verify function returns False - 4. Verify error is logged - -#### TC-039: Directory Creation -- **Description:** Test automatic directory creation -- **Input:** Valid document with non-existent program directory -- **Expected Output:** `True` -- **Test Steps:** - 1. Ensure program directory doesn't exist - 2. Call download function - 3. Verify directory is created - 4. Verify file is saved - ---- - -### 8. `generate_filename(doc: Dict, program_type: str) -> Optional[str]` - -**Function Description:** Generates standardized filename for a document. - -**Test Cases:** - -#### TC-040: MPFS Final Rule Filename -- **Description:** Test filename generation for MPFS final rule -- **Input:** - - `doc={"document_number": "2024-06431", "title": "Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update", "type": "Rule"}` - - `program_type="MPFS"` -- **Expected Output:** `"2025_MPFS_final_2024-06431.xml"` - -#### TC-041: HOSPICE Proposed Rule Filename -- **Description:** Test filename generation for HOSPICE proposed rule -- **Input:** - - `doc={"document_number": "2024-06432", "title": "Medicare Program; Fiscal Year (FY) 2025 Hospice Wage Index Update", "type": "Proposed Rule"}` - - `program_type="HOSPICE"` -- **Expected Output:** `"2025_HOSPICE_proposed_2024-06432.xml"` - -#### TC-042: SNF Final Rule Filename -- **Description:** Test filename generation for SNF final rule -- **Input:** - - `doc={"document_number": "2024-06433", "title": "Medicare Program; Federal Fiscal Year 2025 Prospective Payment System and Consolidated Billing for Skilled Nursing Facilities", "type": "Rule"}` - - `program_type="SNF"` -- **Expected Output:** `"2025_SNF_final_2024-06433.xml"` - -#### TC-043: Missing Document Number -- **Description:** Test with missing document number -- **Input:** - - `doc={"title": "Test Document", "type": "Rule"}` - - `program_type="MPFS"` -- **Expected Output:** `None` - -#### TC-044: Missing Document Type -- **Description:** Test with missing document type -- **Input:** - - `doc={"document_number": "2024-06431", "title": "Test Document"}` - - `program_type="MPFS"` -- **Expected Output:** `None` - -#### TC-045: Year Extraction Failure -- **Description:** Test when year cannot be extracted from title -- **Input:** - - `doc={"document_number": "2024-06431", "title": "Medicare Program Update", "type": "Rule"}` - - `program_type="MPFS"` -- **Expected Output:** `None` - ---- - -### 9. `parse_args() -> argparse.Namespace` - -**Function Description:** Parses command line arguments for the script. - -**Test Cases:** - -#### TC-046: Default Arguments -- **Description:** Test with no command line arguments -- **Input:** `[]` (empty argument list) -- **Expected Output:** Namespace with default values -- **Expected Values:** - - mode='latest' - - doc_number=None - - date=None - - days=365 - - output_dir='data' - - verbose=False - -#### TC-047: Single Mode with Document Number -- **Description:** Test single mode with document number -- **Input:** `["--mode", "single", "--doc-number", "2024-06431"]` -- **Expected Output:** Namespace with single mode and document number -- **Expected Values:** - - mode='single' - - doc_number='2024-06431' - -#### TC-048: Latest Mode with Custom Days -- **Description:** Test latest mode with custom days -- **Input:** `["--mode", "latest", "--days", "30"]` -- **Expected Output:** Namespace with latest mode and 30 days -- **Expected Values:** - - mode='latest' - - days=30 - -#### TC-049: Verbose Mode -- **Description:** Test verbose flag -- **Input:** `["--verbose"]` -- **Expected Output:** Namespace with verbose=True - -#### TC-050: Custom Output Directory -- **Description:** Test custom output directory -- **Input:** `["--output-dir", "/custom/path"]` -- **Expected Output:** Namespace with custom output directory -- **Expected Values:** - - output_dir='/custom/path' - -#### TC-051: Invalid Mode -- **Description:** Test with invalid mode argument -- **Input:** `["--mode", "invalid"]` -- **Expected Output:** SystemExit exception -- **Test Steps:** - 1. Call parse_args with invalid mode - 2. Verify SystemExit is raised - ---- - -### 10. `main()` - -**Function Description:** Main function that orchestrates the complete document fetching process. - -**Test Cases:** - -#### TC-052: Single Document Mode -- **Description:** Test main function in single document mode -- **Input:** Command line arguments for single document -- **Expected Output:** Downloaded XML file -- **Test Steps:** - 1. Mock command line arguments for single mode - 2. Call main function - 3. Verify document is fetched and downloaded - 4. Verify summary is printed - -#### TC-053: Latest Documents Mode -- **Description:** Test main function in latest documents mode -- **Input:** Command line arguments for latest documents -- **Expected Output:** Multiple downloaded XML files -- **Test Steps:** - 1. Mock command line arguments for latest mode - 2. Call main function - 3. Verify documents are fetched and downloaded - 4. Verify summary statistics are printed - -#### TC-054: Error Handling -- **Description:** Test main function error handling -- **Input:** Invalid command line arguments -- **Expected Output:** Error message and graceful exit -- **Test Steps:** - 1. Mock invalid arguments - 2. Call main function - 3. Verify error is handled gracefully - -#### TC-055: Summary Statistics -- **Description:** Test summary statistics generation -- **Input:** Various document processing scenarios -- **Expected Output:** Accurate summary statistics -- **Test Steps:** - 1. Process documents with various outcomes - 2. Verify summary shows correct counts - 3. Verify all categories are accounted for - ---- - -## Module: `app/core/auto_update_pipeline.py` - -### 11. `AutoUpdatePipeline.__init__(days_back: int = 365, model: str = None)` - -**Function Description:** Initializes the AutoUpdatePipeline with configuration parameters. - -**Test Cases:** - -#### TC-056: Default Initialization -- **Description:** Test initialization with default parameters -- **Input:** No parameters -- **Expected Output:** AutoUpdatePipeline instance with default values -- **Expected Values:** - - days_back=365 - - model=config.default_embedding_model - - data_dir=Path(config.docs_data_path) - -#### TC-057: Custom Days Back -- **Description:** Test initialization with custom days_back -- **Input:** `days_back=30` -- **Expected Output:** AutoUpdatePipeline instance with custom days_back -- **Expected Values:** - - days_back=30 - -#### TC-058: Custom Model -- **Description:** Test initialization with custom model -- **Input:** `model="text-embedding-ada-002"` -- **Expected Output:** AutoUpdatePipeline instance with custom model -- **Expected Values:** - - model="text-embedding-ada-002" - -#### TC-059: Both Custom Parameters -- **Description:** Test initialization with both custom parameters -- **Input:** `days_back=60, model="text-embedding-ada-002"` -- **Expected Output:** AutoUpdatePipeline instance with both custom values -- **Expected Values:** - - days_back=60 - - model="text-embedding-ada-002" - ---- - -### 12. `fetch_new_regulations() -> List[Dict]` - -**Function Description:** Fetches new regulations from Federal Register and filters for relevant documents. - -**Test Cases:** - -#### TC-060: Successful Fetch with Relevant Documents -- **Description:** Test fetching regulations with relevant documents found -- **Input:** None (uses instance days_back) -- **Expected Output:** List of relevant regulation documents -- **Test Steps:** - 1. Mock get_latest_documents to return relevant documents - 2. Call fetch_new_regulations - 3. Verify relevant documents are returned - 4. Verify filtering logic works correctly - -#### TC-061: No Relevant Documents -- **Description:** Test when no relevant documents are found -- **Input:** None -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Mock get_latest_documents to return irrelevant documents - 2. Call fetch_new_regulations - 3. Verify empty list is returned - -#### TC-062: Correction Document Filtering -- **Description:** Test filtering of correction documents -- **Input:** Documents including correction documents -- **Expected Output:** List excluding correction documents -- **Test Steps:** - 1. Mock documents including correction documents (starting with "C") - 2. Call fetch_new_regulations - 3. Verify correction documents are filtered out - -#### TC-063: Future-Dated Document Filtering -- **Description:** Test filtering of future-dated documents -- **Input:** Documents including future-dated documents -- **Expected Output:** List excluding future-dated documents -- **Test Steps:** - 1. Mock documents including future dates - 2. Call fetch_new_regulations - 3. Verify future-dated documents are filtered out - -#### TC-064: Non-Rule Document Filtering -- **Description:** Test filtering of non-rule documents -- **Input:** Documents including non-rule types -- **Expected Output:** List containing only Rule and Proposed Rule documents -- **Test Steps:** - 1. Mock documents including non-rule types - 2. Call fetch_new_regulations - 3. Verify only rule documents are included - -#### TC-065: Unrecognized Program Type Filtering -- **Description:** Test filtering of documents with unrecognized program types -- **Input:** Documents including unrecognized program types -- **Expected Output:** List containing only recognized program types -- **Test Steps:** - 1. Mock documents including unrecognized program types - 2. Call fetch_new_regulations - 3. Verify only recognized program types are included - -#### TC-066: Error Handling -- **Description:** Test error handling during fetch process -- **Input:** None (with mocked error) -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Mock get_latest_documents to raise exception - 2. Call fetch_new_regulations - 3. Verify empty list is returned - 4. Verify error is logged - ---- - -### 13. `download_new_files(regulations: List[Dict]) -> List[Path]` - -**Function Description:** Downloads XML files for new regulations. - -**Test Cases:** - -#### TC-067: Successful Downloads -- **Description:** Test successful download of multiple files -- **Input:** List of valid regulation documents -- **Expected Output:** List of downloaded file paths -- **Test Steps:** - 1. Mock valid regulation documents - 2. Call download_new_files - 3. Verify files are downloaded - 4. Verify correct file paths are returned - -#### TC-068: Empty Regulations List -- **Description:** Test with empty regulations list -- **Input:** `[]` -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Call download_new_files with empty list - 2. Verify empty list is returned - 3. Verify appropriate log message - -#### TC-069: File Already Exists -- **Description:** Test when files already exist and are valid -- **Input:** Regulations for files that already exist -- **Expected Output:** List of existing file paths -- **Test Steps:** - 1. Create valid XML files at expected locations - 2. Call download_new_files - 3. Verify existing files are not re-downloaded - 4. Verify file paths are returned - -#### TC-070: Download Failure Handling -- **Description:** Test handling of download failures -- **Input:** Regulations including some that will fail to download -- **Expected Output:** List of successfully downloaded file paths -- **Test Steps:** - 1. Mock some downloads to fail - 2. Call download_new_files - 3. Verify successful downloads are returned - 4. Verify failed downloads are logged - -#### TC-071: Filename Generation Failure -- **Description:** Test when filename generation fails -- **Input:** Regulations that cannot generate valid filenames -- **Expected Output:** List excluding failed filename generations -- **Test Steps:** - 1. Mock regulations that cause filename generation to fail - 2. Call download_new_files - 3. Verify failed cases are skipped - 4. Verify errors are logged - -#### TC-072: Directory Creation -- **Description:** Test automatic directory creation for new program types -- **Input:** Regulations for new program types -- **Expected Output:** List of downloaded files in new directories -- **Test Steps:** - 1. Mock regulations for new program types - 2. Ensure directories don't exist - 3. Call download_new_files - 4. Verify directories are created - 5. Verify files are saved in correct locations - ---- - -### 14. `process_new_files(downloaded_files: List[Path]) -> List[Dict]` - -**Function Description:** Processes newly downloaded files through incremental pipeline. - -**Test Cases:** - -#### TC-073: Successful Processing -- **Description:** Test successful processing of multiple files -- **Input:** List of valid file paths -- **Expected Output:** List of processing results -- **Test Steps:** - 1. Mock valid file paths - 2. Mock incremental pipeline to return success results - 3. Call process_new_files - 4. Verify processing results are returned - -#### TC-074: Empty Files List -- **Description:** Test with empty files list -- **Input:** `[]` -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Call process_new_files with empty list - 2. Verify empty list is returned - 3. Verify appropriate log message - -#### TC-075: Processing Failure Handling -- **Description:** Test handling of processing failures -- **Input:** Files including some that will fail processing -- **Expected Output:** List of processing results (successful and failed) -- **Test Steps:** - 1. Mock some files to fail processing - 2. Call process_new_files - 3. Verify successful processing results are returned - 4. Verify failed processing is logged - -#### TC-076: Path Conversion -- **Description:** Test conversion of absolute paths to relative paths -- **Input:** List of absolute file paths -- **Expected Output:** Processing results for relative paths -- **Test Steps:** - 1. Provide absolute file paths - 2. Call process_new_files - 3. Verify paths are converted to relative paths - 4. Verify incremental pipeline receives relative paths - -#### TC-077: Cost Tracking -- **Description:** Test cost tracking in processing results -- **Input:** Files with various processing costs -- **Expected Output:** Processing results with cost information -- **Test Steps:** - 1. Mock processing results with different costs - 2. Call process_new_files - 3. Verify cost information is preserved - 4. Verify cost display logic works correctly - ---- - -### 15. `run_full_update() -> Dict` - -**Function Description:** Runs the complete automated update process. - -**Test Cases:** - -#### TC-078: Complete Successful Update -- **Description:** Test complete successful update process -- **Input:** None -- **Expected Output:** Dictionary with complete update statistics -- **Test Steps:** - 1. Mock all sub-functions to return success - 2. Call run_full_update - 3. Verify all steps are executed - 4. Verify comprehensive statistics are returned - 5. Verify timing information is included - -#### TC-079: No New Regulations -- **Description:** Test when no new regulations are found -- **Input:** None -- **Expected Output:** Dictionary with zero statistics -- **Test Steps:** - 1. Mock fetch_new_regulations to return empty list - 2. Call run_full_update - 3. Verify statistics show zero values - 4. Verify process completes gracefully - -#### TC-080: Partial Success -- **Description:** Test when some steps succeed and others fail -- **Input:** None -- **Expected Output:** Dictionary with mixed success/failure statistics -- **Test Steps:** - 1. Mock mixed success/failure scenarios - 2. Call run_full_update - 3. Verify statistics reflect partial success - 4. Verify all steps are attempted - -#### TC-081: Error Handling -- **Description:** Test error handling during update process -- **Input:** None (with mocked errors) -- **Expected Output:** Dictionary with error information -- **Test Steps:** - 1. Mock errors in various steps - 2. Call run_full_update - 3. Verify errors are handled gracefully - 4. Verify partial results are returned - -#### TC-082: Statistics Calculation -- **Description:** Test accurate statistics calculation -- **Input:** Various processing scenarios -- **Expected Output:** Accurate statistics dictionary -- **Expected Statistics:** - - regulations_found - - files_downloaded - - files_processed - - files_successful - - total_chunks_created - - total_embeddings_added - - total_cost - - duration_seconds - -#### TC-083: Timing Information -- **Description:** Test timing information in results -- **Input:** None -- **Expected Output:** Dictionary with accurate timing -- **Test Steps:** - 1. Call run_full_update - 2. Verify duration_seconds is included - 3. Verify timing is reasonable - 4. Verify timing is rounded to 2 decimal places - ---- - -### 16. `check_for_updates() -> bool` - -**Function Description:** Checks if there are any new regulations available. - -**Test Cases:** - -#### TC-084: Updates Available -- **Description:** Test when new regulations are available -- **Input:** None -- **Expected Output:** `True` -- **Test Steps:** - 1. Mock fetch_new_regulations to return new regulations - 2. Mock some files to not exist - 3. Call check_for_updates - 4. Verify True is returned - -#### TC-085: No Updates Available -- **Description:** Test when no new regulations are available -- **Input:** None -- **Expected Output:** `False` -- **Test Steps:** - 1. Mock fetch_new_regulations to return empty list - 2. Call check_for_updates - 3. Verify False is returned - -#### TC-086: All Files Already Exist -- **Description:** Test when regulations exist but files are already downloaded -- **Input:** None -- **Expected Output:** `False` -- **Test Steps:** - 1. Mock fetch_new_regulations to return regulations - 2. Mock all files to exist - 3. Call check_for_updates - 4. Verify False is returned - -#### TC-087: Mixed Scenario -- **Description:** Test when some files exist and others don't -- **Input:** None -- **Expected Output:** `True` -- **Test Steps:** - 1. Mock fetch_new_regulations to return regulations - 2. Mock some files to exist and others to not exist - 3. Call check_for_updates - 4. Verify True is returned (if any files missing) - -#### TC-088: Error Handling -- **Description:** Test error handling during update check -- **Input:** None (with mocked errors) -- **Expected Output:** `False` -- **Test Steps:** - 1. Mock fetch_new_regulations to raise exception - 2. Call check_for_updates - 3. Verify False is returned - 4. Verify error is logged - ---- - -### 17. `get_system_status() -> Dict` - -**Function Description:** Gets comprehensive system status information. - -**Test Cases:** - -#### TC-089: Complete Status Information -- **Description:** Test complete system status retrieval -- **Input:** None -- **Expected Output:** Dictionary with comprehensive status information -- **Expected Keys:** - - updates_available - - days_back - - last_check - - processed_files_count - - total_chunks - - faiss_index_size - - new_files - - deleted_files - -#### TC-090: Status with Updates Available -- **Description:** Test status when updates are available -- **Input:** None -- **Expected Output:** Dictionary with updates_available=True -- **Test Steps:** - 1. Mock check_for_updates to return True - 2. Call get_system_status - 3. Verify updates_available=True - -#### TC-091: Status with No Updates -- **Description:** Test status when no updates are available -- **Input:** None -- **Expected Output:** Dictionary with updates_available=False -- **Test Steps:** - 1. Mock check_for_updates to return False - 2. Call get_system_status - 3. Verify updates_available=False - -#### TC-092: Incremental Pipeline Status Integration -- **Description:** Test integration with incremental pipeline status -- **Input:** None -- **Expected Output:** Dictionary including incremental pipeline status -- **Test Steps:** - 1. Mock incremental pipeline status - 2. Call get_system_status - 3. Verify incremental pipeline status is included - 4. Verify all status information is merged correctly - -#### TC-093: Timestamp Information -- **Description:** Test timestamp information in status -- **Input:** None -- **Expected Output:** Dictionary with current timestamp -- **Test Steps:** - 1. Call get_system_status - 2. Verify last_check contains current timestamp - 3. Verify timestamp is in ISO format - ---- - -## Integration Test Cases - -### TC-094: End-to-End Pipeline Test -**Description:** Test complete end-to-end pipeline execution -**Test Steps:** -1. Initialize AutoUpdatePipeline -2. Run complete update process -3. Verify all components work together -4. Verify final results are consistent - -### TC-095: Error Recovery Test -**Description:** Test system recovery after errors -**Test Steps:** -1. Introduce errors at various points -2. Verify system continues processing -3. Verify partial results are preserved -4. Verify error reporting is accurate - -### TC-096: Performance Test -**Description:** Test performance with large datasets -**Test Steps:** -1. Use large number of regulations -2. Measure processing time -3. Verify memory usage is reasonable -4. Verify system remains responsive - -### TC-097: Configuration Test -**Description:** Test different configuration scenarios -**Test Steps:** -1. Test with different models -2. Test with different days_back values -3. Test with different data directories -4. Verify configuration is properly applied - ---- - -## Test Data Requirements - -### Sample Document Data -```json -{ - "document_number": "2024-06431", - "title": "Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update", - "type": "Rule", - "publication_date": "2024-03-28", - "html_url": "https://www.federalregister.gov/documents/2024/03/28/2024-06431/...", - "pdf_url": "https://www.govinfo.gov/content/pkg/FR-2024-03-28/pdf/2024-06431.pdf", - "full_text_xml_url": "https://www.federalregister.gov/documents/full_text/xml/2024/2024-06431.xml" -} -``` - -### Sample XML Structure -```xml - - - - 2024-06431 - 2024-03-28 - Rule - Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update - - - Centers for Medicare & Medicaid Services (CMS), HHS. - Final rule. - - - Supplementary Information: - I. Executive Summary -

...

- - - - -``` - -### Sample Processing Results -```json -{ - "status": "success", - "chunks_created": 150, - "embeddings_added": 150, - "estimated_cost": 0.0450, - "file_path": "MPFS/2025_MPFS_final_2024-06431.xml" -} -``` - ---- - -## Notes - -1. **Mocking Strategy:** All external API calls should be mocked to ensure test reliability and speed. -2. **File System Operations:** Use temporary directories for file system tests to avoid conflicts. -3. **Error Simulation:** Test various error conditions including network failures, invalid data, and system errors. -4. **Performance Considerations:** Monitor test execution time and resource usage for large datasets. -5. **Configuration Management:** Ensure tests use appropriate configuration for different environments. -6. **Logging Verification:** Verify that appropriate log messages are generated for different scenarios. -7. **Edge Cases:** Pay special attention to edge cases such as empty inputs, malformed data, and boundary conditions. \ No newline at end of file diff --git a/test_cases_incremental_chunking_embedding.md b/test_cases_incremental_chunking_embedding.md deleted file mode 100644 index e5c69d5c..00000000 --- a/test_cases_incremental_chunking_embedding.md +++ /dev/null @@ -1,1164 +0,0 @@ -# Test Cases for Incremental Chunking and Embedding Functions - -**Author:** Fanxing Bu -**Module:** `app/core/incremental_chunker.py`, `app/core/incremental_faiss.py`, `app/core/xml_chunker.py`, `app/core/build_faiss.py` - -## Overview - -This document provides comprehensive test cases for the incremental processing system, including: -- Incremental chunking functions -- Incremental embedding functions -- Base chunking and embedding functions -- Integration between components - ---- - -## Module: `app/core/xml_chunker.py` (Base Chunking) - -### 1. `XMLChunker.__init__(input_dir, chunk_words, overlap_sentences, output_chunks)` - -**Function Description:** Initializes the XML chunker with configuration parameters. - -**Test Cases:** - -#### TC-001: Default Initialization -- **Description:** Test initialization with default parameters -- **Input:** No parameters -- **Expected Output:** XMLChunker instance with default values -- **Expected Values:** - - input_dir=config.docs_data_path - - chunk_words=500 - - overlap_sentences=1 - - output_chunks=config.build_faiss_output_folder/chunks.json - -#### TC-002: Custom Parameters -- **Description:** Test initialization with custom parameters -- **Input:** `input_dir="/custom/path", chunk_words=300, overlap_sentences=2, output_chunks="/custom/chunks.json"` -- **Expected Output:** XMLChunker instance with custom values -- **Expected Values:** - - input_dir=Path("/custom/path") - - chunk_words=300 - - overlap_sentences=2 - - output_chunks="/custom/chunks.json" - ---- - -### 2. `clean_text(text: str) -> str` - -**Function Description:** Cleans text by removing extra whitespace. - -**Test Cases:** - -#### TC-003: Normal Text Cleaning -- **Description:** Test cleaning of normal text with extra whitespace -- **Input:** `" This is a test text "` -- **Expected Output:** `"This is a test text"` - -#### TC-004: None Input -- **Description:** Test handling of None input -- **Input:** `None` -- **Expected Output:** `""` - -#### TC-005: Empty String -- **Description:** Test handling of empty string -- **Input:** `""` -- **Expected Output:** `""` - -#### TC-006: Multiple Whitespace Characters -- **Description:** Test cleaning of various whitespace characters -- **Input:** `"Text\twith\nmultiple\r\nspaces"` -- **Expected Output:** `"Text with multiple spaces"` - ---- - -### 3. `infer_metadata_from_filename(filename: str) -> Dict` - -**Function Description:** Extracts metadata from filename using regex patterns. - -**Test Cases:** - -#### TC-007: MPFS Final Rule -- **Description:** Test MPFS final rule filename parsing -- **Input:** `"2025_MPFS_final_2024-06431.xml"` -- **Expected Output:** -```json -{ - "source_file": "2025_MPFS_final_2024-06431.xml", - "program": "MPFS", - "rule_type": "Final", - "year": 2025 -} -``` - -#### TC-008: HOSPICE Proposed Rule -- **Description:** Test HOSPICE proposed rule filename parsing -- **Input:** `"2025_HOSPICE_proposed_2024-06432.xml"` -- **Expected Output:** -```json -{ - "source_file": "2025_HOSPICE_proposed_2024-06432.xml", - "program": "Hospice", - "rule_type": "Proposed", - "year": 2025 -} -``` - -#### TC-009: SNF Final Rule -- **Description:** Test SNF final rule filename parsing -- **Input:** `"2025_SNF_final_2024-06433.xml"` -- **Expected Output:** -```json -{ - "source_file": "2025_SNF_final_2024-06433.xml", - "program": "SNF", - "rule_type": "Final", - "year": 2025 -} -``` - -#### TC-010: Unknown Program Type -- **Description:** Test filename with unknown program type -- **Input:** `"2025_UNKNOWN_final_2024-06434.xml"` -- **Expected Output:** -```json -{ - "source_file": "2025_UNKNOWN_final_2024-06434.xml", - "program": "Unknown", - "rule_type": "Final", - "year": 2025 -} -``` - -#### TC-011: No Year in Filename -- **Description:** Test filename without year -- **Input:** `"MPFS_final_2024-06431.xml"` -- **Expected Output:** -```json -{ - "source_file": "MPFS_final_2024-06431.xml", - "program": "MPFS", - "rule_type": "Final", - "year": null -} -``` - ---- - -### 4. `extract_preamb_metadata(root: ET.Element) -> Dict` - -**Function Description:** Extracts metadata from XML document preamble. - -**Test Cases:** - -#### TC-012: Complete Metadata Extraction -- **Description:** Test extraction of complete metadata from XML -- **Input:** XML element with all metadata fields -- **Expected Output:** -```json -{ - "title": "Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update", - "document_id": "2024-06431", - "cfr": "42 CFR 484", - "effective_date": "January 1, 2025" -} -``` - -#### TC-013: Missing Metadata Fields -- **Description:** Test extraction when some fields are missing -- **Input:** XML element with missing fields -- **Expected Output:** -```json -{ - "title": "Medicare Program Update", - "document_id": "", - "cfr": "", - "effective_date": "" -} -``` - -#### TC-014: Empty XML Element -- **Description:** Test extraction from empty XML element -- **Input:** Empty XML element -- **Expected Output:** -```json -{ - "title": "", - "document_id": "", - "cfr": "", - "effective_date": "" -} -``` - ---- - -### 5. `chunk_document(root: ET.Element, metadata: Dict) -> List[Dict]` - -**Function Description:** Chunks XML document into smaller pieces based on word count and section headers. - -**Test Cases:** - -#### TC-015: Single Section Document -- **Description:** Test chunking of document with single section -- **Input:** XML with single section and multiple paragraphs -- **Expected Output:** List of chunks with proper metadata -- **Test Steps:** - 1. Create XML with single section - 2. Add multiple paragraphs exceeding chunk_words - 3. Verify chunks are created with correct word count - 4. Verify section headers are preserved - -#### TC-016: Multi-Section Document -- **Description:** Test chunking of document with multiple sections -- **Input:** XML with multiple sections (HD1, HD2, HD3) -- **Expected Output:** List of chunks with hierarchical section headers -- **Test Steps:** - 1. Create XML with multiple section levels - 2. Verify section hierarchy is maintained - 3. Verify chunks span across sections correctly - -#### TC-017: Overlap Between Chunks -- **Description:** Test sentence overlap between consecutive chunks -- **Input:** XML with paragraphs that create chunks with overlap -- **Expected Output:** Chunks with specified sentence overlap -- **Test Steps:** - 1. Create XML that triggers chunk creation - 2. Verify last sentences of chunk appear in next chunk - 3. Verify overlap_sentences parameter is respected - -#### TC-018: Empty Document -- **Description:** Test chunking of empty document -- **Input:** XML with no content -- **Expected Output:** Empty list `[]` - -#### TC-019: Document Below Chunk Threshold -- **Description:** Test document that doesn't reach chunk threshold -- **Input:** XML with content below chunk_words limit -- **Expected Output:** Single chunk with all content - ---- - -### 6. `process_files() -> List[Dict]` - -**Function Description:** Processes all XML files in input directory. - -**Test Cases:** - -#### TC-020: Multiple Files Processing -- **Description:** Test processing of multiple XML files -- **Input:** Directory with multiple XML files -- **Expected Output:** Combined list of chunks from all files -- **Test Steps:** - 1. Create multiple XML files in input directory - 2. Call process_files - 3. Verify all files are processed - 4. Verify chunks from all files are included - -#### TC-021: Skip Root Files -- **Description:** Test that root-level XML files are skipped -- **Input:** Directory with XML files at root level -- **Expected Output:** Empty list (no chunks from root files) -- **Test Steps:** - 1. Create XML files at root level - 2. Call process_files - 3. Verify root files are skipped - 4. Verify appropriate log messages - -#### TC-022: Error Handling -- **Description:** Test handling of malformed XML files -- **Input:** Directory with valid and invalid XML files -- **Expected Output:** Chunks from valid files only -- **Test Steps:** - 1. Create mix of valid and invalid XML files - 2. Call process_files - 3. Verify valid files are processed - 4. Verify errors are logged for invalid files - ---- - -### 7. `save_chunks(chunks: List[Dict]) -> None` - -**Function Description:** Saves chunks to output file. - -**Test Cases:** - -#### TC-023: Save Valid Chunks -- **Description:** Test saving valid chunks to file -- **Input:** List of valid chunk dictionaries -- **Expected Output:** Chunks saved to output file -- **Test Steps:** - 1. Create valid chunks list - 2. Call save_chunks - 3. Verify file is created - 4. Verify content matches input - -#### TC-024: Create Output Directory -- **Description:** Test automatic creation of output directory -- **Input:** Chunks with non-existent output directory -- **Expected Output:** Directory created and chunks saved -- **Test Steps:** - 1. Set output_chunks to non-existent directory - 2. Call save_chunks - 3. Verify directory is created - 4. Verify file is saved - ---- - -## Module: `app/core/incremental_chunker.py` (Incremental Chunking) - -### 8. `IncrementalChunker.__init__(input_dir, chunk_words, overlap_sentences, output_chunks)` - -**Function Description:** Initializes incremental chunker with configuration. - -**Test Cases:** - -#### TC-025: Default Initialization -- **Description:** Test initialization with default parameters -- **Input:** No parameters -- **Expected Output:** IncrementalChunker instance with defaults -- **Expected Values:** - - input_dir=config.docs_data_path - - chunk_words=500 - - overlap_sentences=1 - - output_chunks=config.build_faiss_output_folder/chunks.json - - processed_files_tracker=config.build_faiss_output_folder/processed_files.json - ---- - -### 9. `get_file_hash(file_path: Path) -> str` - -**Function Description:** Generates SHA256 hash of file content. - -**Test Cases:** - -#### TC-026: File Hash Generation -- **Description:** Test hash generation for valid file -- **Input:** Path to valid file -- **Expected Output:** SHA256 hash string -- **Test Steps:** - 1. Create test file with known content - 2. Call get_file_hash - 3. Verify hash matches expected value - -#### TC-027: Non-existent File -- **Description:** Test hash generation for non-existent file -- **Input:** Path to non-existent file -- **Expected Output:** FileNotFoundError - ---- - -### 10. `load_processed_files() -> Dict` - -**Function Description:** Loads processed files tracking information. - -**Test Cases:** - -#### TC-028: Load Existing File -- **Description:** Test loading existing processed files -- **Input:** Existing processed_files.json -- **Expected Output:** Dictionary with file tracking data -- **Sample Data:** -```json -{ - "MPFS/2025_MPFS_final_2024-06431.xml": { - "hash": "abc123...", - "processed_at": "1703123456.789", - "chunks_count": 150 - } -} -``` - -#### TC-029: Non-existent File -- **Description:** Test loading when file doesn't exist -- **Input:** Non-existent processed_files.json -- **Expected Output:** Empty dictionary `{}` - ---- - -### 11. `save_processed_files(processed_files: Dict) -> None` - -**Function Description:** Saves processed files tracking information. - -**Test Cases:** - -#### TC-030: Save Valid Data -- **Description:** Test saving valid processed files data -- **Input:** Valid dictionary with file tracking data -- **Expected Output:** Data saved to file -- **Test Steps:** - 1. Create valid tracking data - 2. Call save_processed_files - 3. Verify file is created/updated - 4. Verify content matches input - ---- - -### 12. `is_file_processed(file_path: Path) -> bool` - -**Function Description:** Checks if file has been processed and is unchanged. - -**Test Cases:** - -#### TC-031: File Not Processed -- **Description:** Test unprocessed file -- **Input:** Path to unprocessed file -- **Expected Output:** `False` - -#### TC-032: File Processed and Unchanged -- **Description:** Test processed file with same hash -- **Input:** Path to processed file with matching hash -- **Expected Output:** `True` - -#### TC-033: File Processed but Modified -- **Description:** Test processed file with different hash -- **Input:** Path to processed file with different hash -- **Expected Output:** `False` - ---- - -### 13. `find_new_files() -> List[Path]` - -**Function Description:** Finds new or modified files that need processing. - -**Test Cases:** - -#### TC-034: New Files Found -- **Description:** Test finding new files -- **Input:** Directory with new XML files -- **Expected Output:** List of paths to new files -- **Test Steps:** - 1. Add new XML files to directory - 2. Call find_new_files - 3. Verify new files are returned - -#### TC-035: Modified Files Found -- **Description:** Test finding modified files -- **Input:** Directory with modified XML files -- **Expected Output:** List of paths to modified files -- **Test Steps:** - 1. Modify existing XML files - 2. Call find_new_files - 3. Verify modified files are returned - -#### TC-036: No New Files -- **Description:** Test when no new files exist -- **Input:** Directory with all files processed -- **Expected Output:** Empty list `[]` - ---- - -### 14. `find_deleted_files() -> List[str]` - -**Function Description:** Finds files that have been deleted. - -**Test Cases:** - -#### TC-037: Deleted Files Found -- **Description:** Test finding deleted files -- **Input:** processed_files.json with entries for deleted files -- **Expected Output:** List of deleted file paths -- **Test Steps:** - 1. Create processed_files.json with file entries - 2. Delete corresponding files - 3. Call find_deleted_files - 4. Verify deleted files are returned - -#### TC-038: No Deleted Files -- **Description:** Test when no files are deleted -- **Input:** All processed files still exist -- **Expected Output:** Empty list `[]` - ---- - -### 15. `cleanup_deleted_files(deleted_files: List[str]) -> None` - -**Function Description:** Removes metadata for deleted files. - -**Test Cases:** - -#### TC-039: Cleanup Deleted Files -- **Description:** Test cleanup of deleted files metadata -- **Input:** List of deleted file paths -- **Expected Output:** Metadata removed from chunks.json -- **Test Steps:** - 1. Create chunks.json with file metadata - 2. Call cleanup_deleted_files - 3. Verify metadata is removed - 4. Verify processed_files.json is updated - ---- - -### 16. `process_single_file(file_path: Path) -> List[Dict]` - -**Function Description:** Processes a single XML file and returns chunks. - -**Test Cases:** - -#### TC-040: Valid File Processing -- **Description:** Test processing of valid XML file -- **Input:** Path to valid XML file -- **Expected Output:** List of chunks with metadata -- **Test Steps:** - 1. Create valid XML file - 2. Call process_single_file - 3. Verify chunks are created - 4. Verify metadata is correct - -#### TC-041: Root File Processing -- **Description:** Test processing of root-level file -- **Input:** Path to root-level XML file -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Create XML file at root level - 2. Call process_single_file - 3. Verify empty list is returned - 4. Verify appropriate log message - -#### TC-042: File Processing Error -- **Description:** Test handling of file processing errors -- **Input:** Path to malformed XML file -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Create malformed XML file - 2. Call process_single_file - 3. Verify empty list is returned - 4. Verify error is logged - ---- - -### 17. `update_chunks_database(new_chunks: List[Dict], file_path: Path) -> None` - -**Function Description:** Updates chunks database with new chunks. - -**Test Cases:** - -#### TC-043: Add New Chunks -- **Description:** Test adding new chunks to database -- **Input:** New chunks and file path -- **Expected Output:** Database updated with new chunks -- **Test Steps:** - 1. Create new chunks - 2. Call update_chunks_database - 3. Verify chunks are added - 4. Verify processed_files.json is updated - -#### TC-044: Replace Existing Chunks -- **Description:** Test replacing existing chunks for same file -- **Input:** New chunks for already processed file -- **Expected Output:** Old chunks replaced with new ones -- **Test Steps:** - 1. Add chunks for existing file - 2. Call update_chunks_database with new chunks - 3. Verify old chunks are removed - 4. Verify new chunks are added - ---- - -### 18. `process_file_incrementally(file_path: str) -> List[Dict]` - -**Function Description:** Processes a single file incrementally. - -**Test Cases:** - -#### TC-045: New File Processing -- **Description:** Test processing of new file -- **Input:** Path to new XML file -- **Expected Output:** List of chunks and database updated -- **Test Steps:** - 1. Create new XML file - 2. Call process_file_incrementally - 3. Verify chunks are created - 4. Verify database is updated - -#### TC-046: Already Processed File -- **Description:** Test processing of already processed file -- **Input:** Path to already processed file -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Process file once - 2. Call process_file_incrementally again - 3. Verify empty list is returned - 4. Verify appropriate log message - -#### TC-047: Modified File Processing -- **Description:** Test processing of modified file -- **Input:** Path to modified XML file -- **Expected Output:** New chunks and database updated -- **Test Steps:** - 1. Process file once - 2. Modify file content - 3. Call process_file_incrementally - 4. Verify new chunks are created - 5. Verify database is updated - ---- - -## Module: `app/core/build_faiss.py` (Base Embedding) - -### 19. `setup_model(model_name) -> Tuple` - -**Function Description:** Sets up model configuration and tokenizer. - -**Test Cases: - -#### TC-048: Default Model Setup -- **Description:** Test setup with default model -- **Input:** `model_name=None` -- **Expected Output:** Tuple of (encoding, model_config) -- **Test Steps:** - 1. Call setup_model with None - 2. Verify encoding is created - 3. Verify model_config is loaded - -#### TC-049: Custom Model Setup -- **Description:** Test setup with custom model -- **Input:** `model_name="text-embedding-3-small"` -- **Expected Output:** Tuple of (encoding, model_config) -- **Test Steps:** - 1. Call setup_model with custom model - 2. Verify encoding matches model - 3. Verify model_config is correct - -#### TC-050: Invalid Model -- **Description:** Test setup with invalid model -- **Input:** `model_name="invalid-model"` -- **Expected Output:** ValueError -- **Test Steps:** - 1. Call setup_model with invalid model - 2. Verify ValueError is raised - ---- - -### 20. `count_tokens(text, encoding) -> int` - -**Function Description:** Counts tokens in text using specified encoding. - -**Test Cases:** - -#### TC-051: Normal Text Token Count -- **Description:** Test token counting for normal text -- **Input:** `text="Hello world", encoding=cl100k_base` -- **Expected Output:** Integer token count -- **Test Steps:** - 1. Create test text - 2. Call count_tokens - 3. Verify token count is reasonable - -#### TC-052: Empty Text -- **Description:** Test token counting for empty text -- **Input:** `text="", encoding=cl100k_base` -- **Expected Output:** `0` - -#### TC-053: Long Text -- **Description:** Test token counting for long text -- **Input:** Long text string -- **Expected Output:** Large integer token count - ---- - -### 21. `split_into_chunks(text, max_tokens, encoding) -> List[str]` - -**Function Description:** Splits long text into smaller chunks by sentence. - -**Test Cases:** - -#### TC-054: Text Below Limit -- **Description:** Test text below token limit -- **Input:** Short text below max_tokens -- **Expected Output:** List with single chunk -- **Test Steps:** - 1. Create short text - 2. Call split_into_chunks - 3. Verify single chunk is returned - -#### TC-055: Text Above Limit -- **Description:** Test text above token limit -- **Input:** Long text above max_tokens -- **Expected Output:** List with multiple chunks -- **Test Steps:** - 1. Create long text - 2. Call split_into_chunks - 3. Verify multiple chunks are returned - 4. Verify each chunk is within limit - -#### TC-056: Sentence Boundary Splitting -- **Description:** Test splitting at sentence boundaries -- **Input:** Text with multiple sentences -- **Expected Output:** Chunks split at sentence boundaries -- **Test Steps:** - 1. Create text with multiple sentences - 2. Call split_into_chunks - 3. Verify chunks end at sentence boundaries - ---- - -### 22. `get_openai_embeddings(texts, model, encoding) -> Tuple` - -**Function Description:** Generates embeddings for text chunks with batching. - -**Test Cases:** - -#### TC-057: Single Batch Processing -- **Description:** Test processing of small text set -- **Input:** Small list of texts -- **Expected Output:** Tuple of (embeddings, total_tokens) -- **Test Steps:** - 1. Create small text list - 2. Call get_openai_embeddings - 3. Verify embeddings are generated - 4. Verify token count is correct - -#### TC-058: Multiple Batch Processing -- **Description:** Test processing of large text set requiring multiple batches -- **Input:** Large list of texts -- **Expected Output:** Tuple of (embeddings, total_tokens) -- **Test Steps:** - 1. Create large text list - 2. Call get_openai_embeddings - 3. Verify all batches are processed - 4. Verify all embeddings are generated - -#### TC-059: Long Chunk Splitting -- **Description:** Test splitting of long chunks during processing -- **Input:** Texts with some exceeding token limit -- **Expected Output:** Embeddings for all chunks including split ones -- **Test Steps:** - 1. Create texts with some long chunks - 2. Call get_openai_embeddings - 3. Verify long chunks are split - 4. Verify embeddings for all chunks - -#### TC-060: API Error Handling -- **Description:** Test handling of OpenAI API errors -- **Input:** Valid texts with mocked API failure -- **Expected Output:** Exception or error handling -- **Test Steps:** - 1. Mock OpenAI API to fail - 2. Call get_openai_embeddings - 3. Verify error is handled appropriately - ---- - -## Module: `app/core/incremental_faiss.py` (Incremental Embedding) - -### 23. `IncrementalFAISS.__init__(model)` - -**Function Description:** Initializes incremental FAISS updater. - -**Test Cases:** - -#### TC-061: Default Model Initialization -- **Description:** Test initialization with default model -- **Input:** `model=None` -- **Expected Output:** IncrementalFAISS instance with default model -- **Test Steps:** - 1. Call __init__ with None - 2. Verify default model is used - 3. Verify OpenAI client is initialized - 4. Verify tokenizer is set up - -#### TC-062: Custom Model Initialization -- **Description:** Test initialization with custom model -- **Input:** `model="text-embedding-3-small"` -- **Expected Output:** IncrementalFAISS instance with custom model -- **Test Steps:** - 1. Call __init__ with custom model - 2. Verify custom model is used - 3. Verify configuration is loaded - -#### TC-063: Missing API Key -- **Description:** Test initialization without API key -- **Input:** No OPENAI_API_KEY environment variable -- **Expected Output:** ValueError -- **Test Steps:** - 1. Remove OPENAI_API_KEY from environment - 2. Call __init__ - 3. Verify ValueError is raised - ---- - -### 24. `count_tokens(text: str) -> int` - -**Function Description:** Counts tokens in text using instance encoding. - -**Test Cases:** - -#### TC-064: Token Counting -- **Description:** Test token counting with instance encoding -- **Input:** `text="Test text for token counting"` -- **Expected Output:** Integer token count -- **Test Steps:** - 1. Create test text - 2. Call count_tokens - 3. Verify token count is correct - ---- - -### 25. `split_into_chunks(text: str, max_tokens: int) -> List[str]` - -**Function Description:** Splits text into chunks using instance encoding. - -**Test Cases:** - -#### TC-065: Text Splitting -- **Description:** Test text splitting with instance encoding -- **Input:** Long text and max_tokens limit -- **Expected Output:** List of text chunks -- **Test Steps:** - 1. Create long text - 2. Call split_into_chunks - 3. Verify chunks are within limit - 4. Verify sentence boundaries are respected - ---- - -### 26. `get_embeddings_for_chunks(chunks: List[str], model: str) -> List[List[float]]` - -**Function Description:** Generates embeddings for text chunks with robust error handling. - -**Test Cases:** - -#### TC-066: Successful Embedding Generation -- **Description:** Test successful embedding generation -- **Input:** List of valid text chunks -- **Expected Output:** List of embedding vectors -- **Test Steps:** - 1. Create valid text chunks - 2. Call get_embeddings_for_chunks - 3. Verify embeddings are generated - 4. Verify all chunks are processed - -#### TC-067: Rate Limit Handling -- **Description:** Test handling of rate limit errors -- **Input:** Text chunks with mocked rate limit -- **Expected Output:** Embeddings after retry -- **Test Steps:** - 1. Mock rate limit error - 2. Call get_embeddings_for_chunks - 3. Verify exponential backoff is used - 4. Verify embeddings are eventually generated - -#### TC-068: Large Batch Splitting -- **Description:** Test splitting of large batches -- **Input:** Large batch exceeding token limit -- **Expected Output:** Embeddings for all chunks -- **Test Steps:** - 1. Create large batch - 2. Call get_embeddings_for_chunks - 3. Verify batch is split - 4. Verify all chunks are processed - -#### TC-069: Empty Chunks List -- **Description:** Test handling of empty chunks list -- **Input:** Empty list `[]` -- **Expected Output:** Empty list `[]` - -#### TC-070: Failed Embedding Generation -- **Description:** Test handling of embedding generation failure -- **Input:** Text chunks with mocked API failure -- **Expected Output:** `None` -- **Test Steps:** - 1. Mock API to fail consistently - 2. Call get_embeddings_for_chunks - 3. Verify None is returned - 4. Verify error is logged - ---- - -### 27. `load_existing_index() -> Optional[faiss.Index]` - -**Function Description:** Loads existing FAISS index if it exists. - -**Test Cases:** - -#### TC-071: Existing Index Load -- **Description:** Test loading existing index -- **Input:** Existing FAISS index file -- **Expected Output:** FAISS index object -- **Test Steps:** - 1. Create FAISS index file - 2. Call load_existing_index - 3. Verify index is loaded - -#### TC-072: No Existing Index -- **Description:** Test when no index exists -- **Input:** No FAISS index file -- **Expected Output:** `None` -- **Test Steps:** - 1. Ensure no index file exists - 2. Call load_existing_index - 3. Verify None is returned - ---- - -### 28. `load_existing_metadata() -> List[Dict]` - -**Function Description:** Loads existing metadata if it exists. - -**Test Cases:** - -#### TC-073: Existing Metadata Load -- **Description:** Test loading existing metadata -- **Input:** Existing metadata file -- **Expected Output:** List of metadata dictionaries -- **Test Steps:** - 1. Create metadata file - 2. Call load_existing_metadata - 3. Verify metadata is loaded - -#### TC-074: No Existing Metadata -- **Description:** Test when no metadata exists -- **Input:** No metadata file -- **Expected Output:** Empty list `[]` -- **Test Steps:** - 1. Ensure no metadata file exists - 2. Call load_existing_metadata - 3. Verify empty list is returned - ---- - -### 29. `create_new_index(dimension: int) -> faiss.Index` - -**Function Description:** Creates a new FAISS index. - -**Test Cases:** - -#### TC-075: New Index Creation -- **Description:** Test creation of new index -- **Input:** `dimension=1536` -- **Expected Output:** FAISS index with specified dimension -- **Test Steps:** - 1. Call create_new_index - 2. Verify index is created - 3. Verify dimension is correct - ---- - -### 30. `update_index_with_new_chunks(new_chunks: List[Dict]) -> int` - -**Function Description:** Updates FAISS index with new chunks. - -**Test Cases:** - -#### TC-076: New Chunks Addition -- **Description:** Test adding new chunks to index -- **Input:** List of new chunk dictionaries -- **Expected Output:** Number of embeddings added -- **Test Steps:** - 1. Create new chunks - 2. Call update_index_with_new_chunks - 3. Verify embeddings are generated - 4. Verify index is updated - 5. Verify metadata is updated - -#### TC-077: Empty Chunks List -- **Description:** Test with empty chunks list -- **Input:** Empty list `[]` -- **Expected Output:** `0` - -#### TC-078: Long Chunk Processing -- **Description:** Test processing of chunks that need splitting -- **Input:** Chunks with long text -- **Expected Output:** Embeddings for all processed chunks -- **Test Steps:** - 1. Create chunks with long text - 2. Call update_index_with_new_chunks - 3. Verify long chunks are split - 4. Verify all chunks are processed - -#### TC-079: Embedding Generation Failure -- **Description:** Test handling of embedding generation failure -- **Input:** Chunks that cause embedding failure -- **Expected Output:** `0` -- **Test Steps:** - 1. Mock embedding generation to fail - 2. Call update_index_with_new_chunks - 3. Verify 0 is returned - 4. Verify error is logged - ---- - -### 31. `update_metadata_with_new_chunks(new_chunks: List[Dict]) -> None` - -**Function Description:** Updates metadata file with new chunks. - -**Test Cases:** - -#### TC-080: Metadata Update -- **Description:** Test updating metadata with new chunks -- **Input:** List of new chunk dictionaries -- **Expected Output:** Metadata file updated -- **Test Steps:** - 1. Create new chunks - 2. Call update_metadata_with_new_chunks - 3. Verify metadata file is updated - 4. Verify all chunks are included - -#### TC-081: Chunk Splitting Consistency -- **Description:** Test consistency between embedding and metadata chunk splitting -- **Input:** Chunks that need splitting -- **Expected Output:** Metadata matches embedding chunks -- **Test Steps:** - 1. Create chunks that need splitting - 2. Call update_metadata_with_new_chunks - 3. Verify splitting logic matches embedding logic - ---- - -### 32. `process_incremental_update(new_chunks: List[Dict]) -> Dict` - -**Function Description:** Processes incremental update with new chunks. - -**Test Cases:** - -#### TC-082: Successful Incremental Update -- **Description:** Test successful incremental update -- **Input:** List of new chunk dictionaries -- **Expected Output:** Dictionary with update statistics -- **Expected Statistics:** - - new_chunks_processed - - new_embeddings_added - - total_tokens - - estimated_cost - - model_used - - model_pricing - -#### TC-083: Empty Update -- **Description:** Test update with empty chunks list -- **Input:** Empty list `[]` -- **Expected Output:** Dictionary with zero statistics -- **Test Steps:** - 1. Call process_incremental_update with empty list - 2. Verify all statistics are zero - 3. Verify cost calculation is correct - -#### TC-084: Cost Calculation -- **Description:** Test accurate cost calculation -- **Input:** Chunks with known token count -- **Expected Output:** Accurate cost estimate -- **Test Steps:** - 1. Create chunks with known token count - 2. Call process_incremental_update - 3. Verify cost calculation is accurate - ---- - -## Integration Test Cases - -### TC-085: Complete Incremental Pipeline -**Description:** Test complete incremental processing pipeline -**Test Steps:** -1. Initialize IncrementalPipeline -2. Add new XML file -3. Run process_single_file -4. Verify chunks are created -5. Verify embeddings are generated -6. Verify FAISS index is updated -7. Verify metadata is updated - -### TC-086: File Modification Handling -**Description:** Test handling of file modifications -**Test Steps:** -1. Process file initially -2. Modify file content -3. Run incremental processing -4. Verify old chunks are removed -5. Verify new chunks are created -6. Verify index is updated correctly - -### TC-087: Error Recovery -**Description:** Test error recovery in pipeline -**Test Steps:** -1. Introduce errors at various points -2. Run incremental processing -3. Verify partial results are preserved -4. Verify errors are handled gracefully -5. Verify system remains consistent - -### TC-088: Performance Testing -**Description:** Test performance with large datasets -**Test Steps:** -1. Use large number of chunks -2. Measure processing time -3. Verify memory usage is reasonable -4. Verify system remains responsive - ---- - -## Test Data Requirements - -### Sample XML Structure -```xml - - - - 2024-06431 - 2024-03-28 - Rule - Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update - - - Centers for Medicare & Medicaid Services (CMS), HHS. - Final rule. - - - Supplementary Information: - I. Executive Summary -

This is the first paragraph of the executive summary.

-

This is the second paragraph with more content.

- A. Background -

Background information goes here.

- - - - -``` - -### Sample Chunk Structure -```json -{ - "text": "This is the first paragraph of the executive summary. This is the second paragraph with more content.", - "section_header": "I. Executive Summary", - "chunk_index": 0, - "hash": "abc123...", - "metadata": { - "source_file": "2025_MPFS_final_2024-06431.xml", - "program": "MPFS", - "rule_type": "Final", - "year": 2025, - "title": "Medicare Program; Calendar Year (CY) 2025 Home Health Prospective Payment System Rate Update", - "subfolder": "MPFS", - "full_path": "/path/to/file.xml" - } -} -``` - -### Sample Embedding Response -```json -{ - "data": [ - { - "embedding": [0.1, 0.2, 0.3, ...], - "index": 0, - "object": "embedding" - } - ], - "model": "text-embedding-3-small", - "object": "list", - "usage": { - "prompt_tokens": 100, - "total_tokens": 100 - } -} -``` - ---- - -## Notes - -1. **Mocking Strategy:** All external API calls should be mocked for reliable testing -2. **File System Operations:** Use temporary directories for file system tests -3. **Error Simulation:** Test various error conditions including network failures and invalid data -4. **Performance Monitoring:** Track processing time and resource usage for large datasets -5. **Consistency Verification:** Ensure chunk splitting logic is consistent between components -6. **Cost Tracking:** Verify accurate cost calculation and tracking -7. **State Management:** Test proper state management during incremental updates \ No newline at end of file diff --git a/test_token_count.py b/test_token_count.py deleted file mode 100644 index 50a5abd2..00000000 --- a/test_token_count.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 - -import json -import sys -import os - -# Add the current directory to path to import our modules -sys.path.append(os.path.dirname(os.path.abspath(__file__))) - -try: - from incremental_faiss import IncrementalFAISS -except ImportError: - print("Error: Could not import IncrementalFAISS") - sys.exit(1) - -def test_token_counting(): - """Test token counting with actual data""" - print("=== Token Counting Test ===") - - # Initialize the FAISS updater - faiss_updater = IncrementalFAISS() - - # Load chunks - chunks_file = "../../rag_data/chunks.json" - with open(chunks_file, "r") as f: - chunks = json.load(f) - - # Find MPFS 2024-14828 chunks - mpfs_chunks = [c for c in chunks if '2024-14828' in c.get('metadata', {}).get('source_file', '')] - - print(f"Found {len(mpfs_chunks)} chunks for MPFS 2024-14828") - - if not mpfs_chunks: - print("No chunks found!") - return - - # Test token counting on first few chunks - print("\nTesting token counting on first 5 chunks:") - total_tokens = 0 - for i, chunk in enumerate(mpfs_chunks[:5]): - text = chunk['text'] - tokens = faiss_updater.count_tokens(text) - total_tokens += tokens - print(f"Chunk {i+1}: {len(text)} chars, {tokens} tokens, ratio: {tokens/len(text):.3f}") - - print(f"\nFirst 5 chunks total: {total_tokens} tokens") - - # Calculate total tokens for all chunks - print("\nCalculating total tokens for all chunks...") - all_tokens = sum(faiss_updater.count_tokens(chunk['text']) for chunk in mpfs_chunks) - print(f"Total tokens for all {len(mpfs_chunks)} chunks: {all_tokens:,}") - - # Compare with reported value - reported_tokens = 14109286 - print(f"Reported tokens: {reported_tokens:,}") - print(f"Difference: {abs(all_tokens - reported_tokens):,}") - print(f"Match: {abs(all_tokens - reported_tokens) < 1000}") - - # Calculate cost - estimated_cost = all_tokens / 1000 * 0.0001 - print(f"\nEstimated cost: ${estimated_cost:.4f}") - -if __name__ == "__main__": - test_token_counting() \ No newline at end of file diff --git a/token_demo.py b/token_demo.py deleted file mode 100644 index 114251c1..00000000 --- a/token_demo.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -""" -Token获取过程演示 -""" - -import json -import sys -import os - -# 添加当前目录到路径 -sys.path.append(os.path.dirname(os.path.abspath(__file__))) - -def demo_token_counting(): - """演示token计数过程""" - print("=== Token获取过程演示 ===\n") - - # 1. 加载chunks数据 - print("1. 加载chunks数据...") - chunks_file = "../../rag_data/chunks.json" - with open(chunks_file, "r") as f: - chunks = json.load(f) - - # 2. 找到特定文件的chunks - print("2. 找到MPFS 2024-14828文件的chunks...") - mpfs_chunks = [c for c in chunks if '2024-14828' in c.get('metadata', {}).get('source_file', '')] - print(f" 找到 {len(mpfs_chunks)} 个chunks\n") - - # 3. 演示token计数（模拟tiktoken） - print("3. Token计数过程:") - print(" - 使用 tiktoken.encoding_for_model('text-embedding-ada-002')") - print(" - 对每个chunk的text进行encode") - print(" - 计算encode后的长度作为token数量") - print() - - # 4. 显示前几个chunks的详细信息 - print("4. 前3个chunks的详细信息:") - total_chars = 0 - for i, chunk in enumerate(mpfs_chunks[:3]): - text = chunk['text'] - chars = len(text) - total_chars += chars - print(f" Chunk {i+1}:") - print(f" - 字符数: {chars:,}") - print(f" - 文本预览: {text[:100]}...") - print() - - # 5. 计算总token数 - print("5. 总token数计算:") - print(" - 总chunks: 699") - print(" - 总字符数: 70,002,413") - print(" - 平均每chunk字符: 100,146") - print(" - 估算token数: 14,109,286") - print() - - # 6. 成本计算 - print("6. 成本计算:") - tokens = 14109286 - cost = tokens / 1000 * 0.0001 - print(f" - 公式: {tokens:,} / 1000 * 0.0001") - print(f" - 成本: ${cost:.4f}") - print() - - # 7. 技术细节 - print("7. 技术实现细节:") - print(" - Tokenizer: tiktoken (OpenAI官方)") - print(" - 模型: text-embedding-ada-002") - print(" - 方法: encoding.encode(text) -> len()") - print(" - 特点: 与OpenAI API完全一致的tokenization") - print() - - print("=== 总结 ===") - print("Token数量是通过以下步骤获取的:") - print("1. 从chunks.json加载文本数据") - print("2. 使用tiktoken对每个chunk的text进行encode") - print("3. 计算encode后的长度作为token数量") - print("4. 累加所有chunks的token数量") - print("5. 根据OpenAI定价计算成本") - -if __name__ == "__main__": - demo_token_counting() \ No newline at end of file diff --git a/xml_label_analysis_results.json b/xml_label_analysis_results.json deleted file mode 100644 index f39dc210..00000000 --- a/xml_label_analysis_results.json +++ /dev/null @@ -1,1575 +0,0 @@ -{ - "folder_results": { - "HOSPICE": { - "folder_name": "HOSPICE", - "file_count": 8, - "file_labels": { - "2025_HOSPICE_final_2024-16910.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DEPDOC", - "E", - "EFFDATE", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "2026_HOSPICE_proposed_2025-06317.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "ROW", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE", - "TNOTE", - "TTITLE" - ], - "2024_HOSPICE_proposed_2023-06769.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "ROW", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TDESC", - "TITLE", - "TNOTE", - "TTITLE" - ], - "2025_HOSPICE_proposed_2024-06921.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATES", - "DEPDOC", - "E", - "ENT", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "ROW", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE", - "TTITLE" - ], - "2023_HOSPICE_proposed_2022-07030.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "ROW", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE", - "TNOTE", - "TTITLE" - ], - "2024_HOSPICE_final_2023-16116.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DEPDOC", - "E", - "EFFDATE", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "2022_HOSPICE_final_2021-16311.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DATED", - "DEPDOC", - "E", - "EFFDATE", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "2023_HOSPICE_final_2022-16214.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DATED", - "DEPDOC", - "E", - "EFFDATE", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "P", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUBPART", - "SUM", - "SUPLINF", - "TITLE" - ] - }, - "all_labels": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DATES", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUBPART", - "SUM", - "SUPLINF", - "TDESC", - "TITLE", - "TNOTE", - "TTITLE" - ], - "common_labels": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DEPDOC", - "E", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "HD", - "LSTSUB", - "NAME", - "P", - "PREAMB", - "PRTPAGE", - "RIN", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "unique_labels_per_file": { - "2025_HOSPICE_final_2024-16910.xml": 37, - "2026_HOSPICE_proposed_2025-06317.xml": 42, - "2024_HOSPICE_proposed_2023-06769.xml": 47, - "2025_HOSPICE_proposed_2024-06921.xml": 42, - "2023_HOSPICE_proposed_2022-07030.xml": 43, - "2024_HOSPICE_final_2023-16116.xml": 37, - "2022_HOSPICE_final_2021-16311.xml": 39, - "2023_HOSPICE_final_2022-16214.xml": 37 - } - }, - "MPFS": { - "folder_name": "MPFS", - "file_count": 8, - "file_labels": { - "2025_MPFS_proposed_2024-14828.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "CONTENTS", - "DEPDOC", - "E", - "EFFDATE", - "EXTRACT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "NOTE", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "SECHD", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUBPART", - "SUM", - "SUPLINF", - "TITLE" - ], - "2024_MPFS_final_2023-24184.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATES", - "DEPDOC", - "E", - "ENT", - "EXTRACT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LSTSUB", - "NAME", - "NOTE", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE", - "TTITLE" - ], - "2025_MPFS_final_2024-25382.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "CONTENTS", - "DATES", - "DEPDOC", - "E", - "EXTRACT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "NOTE", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "RULE", - "SECHD", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUBPART", - "SUM", - "SUPLINF", - "TITLE" - ], - "2022_MPFS_proposed_2021-14973.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DATED", - "DEPDOC", - "E", - "EFFDATE", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "NOTE", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "2022_MPFS_final_2021-23972.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "EXTRACT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "NOTE", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE", - "TTITLE" - ], - "2024_MPFS_proposed_2023-14624.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DEPDOC", - "E", - "EFFDATE", - "EXTRACT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "2023_MPFS_proposed_2022-14562.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "APPENDIX", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DATES", - "DEPDOC", - "E", - "ENT", - "EXTRACT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LSTSUB", - "NAME", - "NOTE", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "ROW", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUBPART", - "SUM", - "SUPLINF", - "TITLE", - "TTITLE" - ], - "2023_MPFS_final_2022-23873.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DATES", - "DEPDOC", - "E", - "ENT", - "EXTRACT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LSTSUB", - "NAME", - "NOTE", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE", - "TNOTE", - "TTITLE" - ] - }, - "all_labels": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "APPENDIX", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "CONTENTS", - "DATED", - "DATES", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "EXTRACT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "NOTE", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECHD", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUBPART", - "SUM", - "SUPLINF", - "TITLE", - "TNOTE", - "TTITLE" - ], - "common_labels": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DEPDOC", - "E", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "RIN", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "unique_labels_per_file": { - "2025_MPFS_proposed_2024-14828.xml": 43, - "2024_MPFS_final_2023-24184.xml": 46, - "2025_MPFS_final_2024-25382.xml": 43, - "2022_MPFS_proposed_2021-14973.xml": 40, - "2022_MPFS_final_2021-23972.xml": 47, - "2024_MPFS_proposed_2023-14624.xml": 39, - "2023_MPFS_proposed_2022-14562.xml": 49, - "2023_MPFS_final_2022-23873.xml": 48 - } - }, - "SNF": { - "folder_name": "SNF", - "file_count": 7, - "file_labels": { - "2023_SNF_final_2022-16457.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DEPDOC", - "E", - "EFFDATE", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "2023_SNF_proposed_2022-07906.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DATED", - "DATES", - "DEPDOC", - "E", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "2025_SNF_proposed_2024-06812.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "ROW", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TDESC", - "TITLE", - "TNOTE", - "TTITLE" - ], - "2024_SNF_final_2023-16249.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATES", - "DEPDOC", - "E", - "ENT", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE", - "TNOTE", - "TTITLE" - ], - "2026_SNF_proposed_2025-06348.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "ROW", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE", - "TNOTE", - "TTITLE" - ], - "2024_SNF_proposed_2023-07137.xml": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "RIN", - "ROW", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TDESC", - "TITLE", - "TNOTE", - "TTITLE" - ], - "2025_SNF_final_2024-16907.xml": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DEPDOC", - "E", - "EFFDATE", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "REGTEXT", - "RIN", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ] - }, - "all_labels": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DATES", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TDESC", - "TITLE", - "TNOTE", - "TTITLE" - ], - "common_labels": [ - "ACT", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "CFR", - "DEPDOC", - "E", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "HD", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRTPAGE", - "RIN", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE" - ], - "unique_labels_per_file": { - "2023_SNF_final_2022-16457.xml": 38, - "2023_SNF_proposed_2022-07906.xml": 39, - "2025_SNF_proposed_2024-06812.xml": 45, - "2024_SNF_final_2023-16249.xml": 46, - "2026_SNF_proposed_2025-06348.xml": 44, - "2024_SNF_proposed_2023-07137.xml": 48, - "2025_SNF_final_2024-16907.xml": 38 - } - } - }, - "all_folder_labels": { - "HOSPICE": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DATES", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUBPART", - "SUM", - "SUPLINF", - "TDESC", - "TITLE", - "TNOTE", - "TTITLE" - ], - "MPFS": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "APPENDIX", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "CONTENTS", - "DATED", - "DATES", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "EXTRACT", - "FP", - "FR", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "NOTE", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECHD", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUBPART", - "SUM", - "SUPLINF", - "TITLE", - "TNOTE", - "TTITLE" - ], - "SNF": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DATES", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "EXTRACT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TDESC", - "TITLE", - "TNOTE", - "TTITLE" - ] - }, - "common_across_folders": [ - "ACT", - "ADD", - "AGENCY", - "AGY", - "AMDPAR", - "AUTH", - "BILCOD", - "BOXHD", - "CFR", - "CHED", - "DATED", - "DATES", - "DEPDOC", - "E", - "EFFDATE", - "ENT", - "FP", - "FRDOC", - "FTNT", - "FTREF", - "FURINF", - "GID", - "GPH", - "GPOTABLE", - "HD", - "LI", - "LSTSUB", - "NAME", - "P", - "PART", - "PREAMB", - "PRORULE", - "PRTPAGE", - "REGTEXT", - "RIN", - "ROW", - "RULE", - "SECTION", - "SECTNO", - "SIG", - "STARS", - "SU", - "SUBAGY", - "SUBJECT", - "SUM", - "SUPLINF", - "TITLE", - "TNOTE", - "TTITLE" - ], - "unique_per_folder": { - "HOSPICE": [], - "MPFS": [ - "APPENDIX", - "CONTENTS", - "NOTE", - "SECHD" - ], - "SNF": [] - }, - "consistency_check": { - "all_folders_have_same_labels": false, - "label_sets_are_equal": false - } -} \ No newline at end of file