Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -239,10 +239,7 @@ yarn-error.log*
faiss.index
rag_data/chunks.json
app/core/rag_data/chunks.json
<<<<<<< HEAD
=======
summary_outputs/
>>>>>>> dev
*.log
old_front/
*.zip
Expand Down
Binary file removed .test_filename_generation.py.icloud
Binary file not shown.
3 changes: 0 additions & 3 deletions app/config/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,6 @@ def overlap_sentences(self):
"""Get overlap sentences from config."""
return self.config['chunking']['overlap_sentences']

<<<<<<< HEAD
=======
# Regulation fetch configuration
@property
def regulation_fetch_days_back(self):
Expand All @@ -147,5 +145,4 @@ def summary_output_dir(self):
rel_path = self.config['summary']['output_dir']
return str(project_root / rel_path)

>>>>>>> dev
config = Config()
3 changes: 0 additions & 3 deletions app/config/development.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@ chunking:
chunk_words: 500
overlap_sentences: 1

<<<<<<< HEAD
=======
# Regulation fetching configuration
regulation_fetch:
days_back: 30 # Default days to look back for new regulations
Expand All @@ -41,7 +39,6 @@ regulation_fetch:
summary:
output_dir: summary_outputs

>>>>>>> dev
# Embedding model configuration
embedding:
# Default model to use for embeddings
Expand Down
3 changes: 0 additions & 3 deletions app/config/production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ build_faiss:
docs_data:
path: data/

<<<<<<< HEAD
=======
# Chunking configuration
chunking:
chunk_words: 500
Expand All @@ -34,7 +32,6 @@ regulation_fetch:
summary:
output_dir: summary_outputs

>>>>>>> dev
# Embedding model configuration
embedding:
# Default model to use for embeddings
Expand Down
3 changes: 0 additions & 3 deletions app/config/production.yml.example
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ build_faiss:
docs_data:
path: data/

<<<<<<< HEAD
=======
# Chunking configuration
chunking:
chunk_words: 500
Expand All @@ -34,7 +32,6 @@ regulation_fetch:
summary:
output_dir: summary_outputs

>>>>>>> dev
# Embedding model configuration
embedding:
# Default model to use for embeddings
Expand Down
Binary file removed app/core/.example_usage.py.icloud
Binary file not shown.
35 changes: 0 additions & 35 deletions app/core/auto_update_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,9 @@
# Add the app directory to Python path for imports
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from config import config
<<<<<<< HEAD

=======
# Configure logging first
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
>>>>>>> dev
# Import pipeline components
from incremental_pipeline import IncrementalPipeline

Expand All @@ -44,19 +40,13 @@
logger.warning(f"⚠️ Data fetcher not available: {e}")
DATA_FETCHER_AVAILABLE = False

<<<<<<< HEAD
# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
=======
# Import incremental summary component
try:
from incremental_summary import IncrementalSummary
SUMMARY_AVAILABLE = True
except ImportError as e:
logger.warning(f"⚠️ Incremental summary not available: {e}")
SUMMARY_AVAILABLE = False
>>>>>>> dev

class AutoUpdatePipeline:
"""
Expand Down Expand Up @@ -85,14 +75,11 @@ def __init__(self, model: str = None, days_back: int = None):
# Initialize core pipeline
self.pipeline = IncrementalPipeline(model=self.model)

<<<<<<< HEAD
=======
# Initialize incremental summary if available
self.summary_manager = None
if SUMMARY_AVAILABLE:
self.summary_manager = IncrementalSummary()

>>>>>>> dev
# Paths
self.data_dir = Path(config.docs_data_path)

Expand All @@ -101,10 +88,7 @@ def __init__(self, model: str = None, days_back: int = None):
logger.info(f"🗓️ Days back: {self.days_back}")
logger.info(f"📁 Data directory: {self.data_dir}")
logger.info(f"🌐 Data fetcher available: {DATA_FETCHER_AVAILABLE}")
<<<<<<< HEAD
=======
logger.info(f"📄 Incremental summary available: {SUMMARY_AVAILABLE}")
>>>>>>> dev

def check_for_new_regulations(self) -> Dict:
"""
Expand Down Expand Up @@ -306,8 +290,6 @@ def run_full_auto_update(self) -> Dict:
# Step 3: Run full incremental update (includes new downloads)
incremental_result = self.pipeline.full_incremental_update()

<<<<<<< HEAD
=======
# Step 4: Run incremental summary update
summary_result = None
if self.summary_manager:
Expand All @@ -317,7 +299,6 @@ def run_full_auto_update(self) -> Dict:
else:
logger.info("📄 Skipping summary update (not available)")

>>>>>>> dev
# Calculate totals
total_cost = incremental_result['total_cost']
end_time = time.time()
Expand All @@ -333,10 +314,7 @@ def run_full_auto_update(self) -> Dict:
'regulations_check': check_result,
'download_result': download_result,
'incremental_result': incremental_result,
<<<<<<< HEAD
=======
'summary_result': summary_result,
>>>>>>> dev
'total_cost': total_cost,
'duration_seconds': round(end_time - start_time, 2),
'status': 'success'
Expand Down Expand Up @@ -364,8 +342,6 @@ def run_incremental_update(self) -> Dict:
try:
result = self.pipeline.full_incremental_update()

<<<<<<< HEAD
=======
# Run incremental summary update
summary_result = None
if self.summary_manager:
Expand All @@ -375,20 +351,15 @@ def run_incremental_update(self) -> Dict:
else:
logger.info("📄 Skipping summary update (not available)")

>>>>>>> dev
logger.info("✅ Incremental update completed!")
logger.info(f" - Files deleted: {len(result['cleanup_result']['deleted_files'])}")
logger.info(f" - Files processed: {len(result['process_result']['processed_files'])}")
logger.info(f" - Total cost: ${result['total_cost']:.4f}")

<<<<<<< HEAD
return result
=======
return {
**result,
'summary_result': summary_result
}
>>>>>>> dev

except Exception as e:
logger.error(f"❌ Incremental update failed: {e}")
Expand Down Expand Up @@ -448,16 +419,13 @@ def process_specific_files(self, file_paths: List[str]) -> Dict:

successful_files = [r['file_path'] for r in results if r['status'] == 'success']

<<<<<<< HEAD
=======
# Generate summaries for processed files
summary_result = None
if self.summary_manager and successful_files:
logger.info("📄 Generating summaries for processed files...")
summary_result = self.summary_manager.generate_summary_for_specific_files(successful_files)
logger.info(f"📄 Summary generation completed: {summary_result['status']}")

>>>>>>> dev
logger.info(f"✅ Specific file processing completed:")
logger.info(f" - Files processed: {len(successful_files)}/{len(resolved_paths)}")
logger.info(f" - Total cost: ${total_cost:.4f}")
Expand All @@ -468,10 +436,7 @@ def process_specific_files(self, file_paths: List[str]) -> Dict:
return {
'processed_files': successful_files,
'failed_files': errors,
<<<<<<< HEAD
=======
'summary_result': summary_result,
>>>>>>> dev
'total_cost': total_cost,
'status': 'success' if not errors else 'partial_success'
}
Expand Down
Loading