diff --git a/Financial Data Fusion and Processing Script b/Financial Data Fusion and Processing Script new file mode 100644 index 0000000..999ffee --- /dev/null +++ b/Financial Data Fusion and Processing Script @@ -0,0 +1,87 @@ +#New idea pick up all the dates , find the most recurrent out of the 30 stocks, sort them datas, create a list. +#Then create a dict out of each csv based on date as main key [close, volume]. Then only pick up the dates +# from your OG list in a loop and store them in a list of df, then concat them and send to excel + +import os +import pandas as pd + +source = r'C:\Users\reyna\Desktop\VAR TEST MONTHLY' #VAR TEST DAILY or VAR TEST WEEKLY or VAR TEST MONTHLY + +output_file = r'C:\Users\reyna\Desktop\fused documents\FusedmonthlyVar.xlsx' #FuseddailyVar or FusedweeklyVar or FusedmonthlyVar +test_file = r'C:\Users\reyna\Desktop\fused documents\test.xlsx' + +""" +#Daily only + +# Initialize an empty list to store DataFrames +dfs = [] + +# Iterate through each file in the directory +for filename in os.listdir(source): + filepath = os.path.join(source, filename) + if os.path.isfile(filepath) and filename.endswith('.csv'): + # Read the CSV file + data = pd.read_csv(filepath) + # Select only the required columns + selected_data = data[['Date']] + # Append selected data to the list of DataFrames + dfs.append(selected_data) + +concatenated_data = pd.concat(dfs, axis= 0) + + +date_counts = concatenated_data['Date'].value_counts() + + + + + +correct_dates = date_counts[date_counts >= 30] #no need for this part for weekly and monthly +correct_dates = correct_dates.sort_index(ascending = True) +correct_dates = correct_dates.to_dict() +key_data = list(correct_dates.keys()) +list_key = pd.DataFrame(key_data) +list_key = list_key.rename(columns = { 0 :"date"}) + + + +#list_key.to_excel(test_file, index=True, header=True) +""" + +df2 = [] +# Iterate through each file in the directory +for filename in os.listdir(source): + filepath = os.path.join(source, filename) + if os.path.isfile(filepath) and filename.endswith('.csv'): + # Read the CSV file + data = pd.read_csv(filepath) + # Select only the required columns + selected_data = data[['Date','Close','Volume']] + selected_data = selected_data.rename(columns={'Date': 'Date'}) + selected_data = selected_data.rename(columns={'Close': 'Close' + filename[:-4]}) + selected_data = selected_data.rename(columns={'Volume': 'Volume' + filename[:-4]}) + # Append selected data to the list of DataFrames + df2.append(selected_data) + +#This part is for monthly and weekly only +concate_data = pd.concat(df2, axis= 1) +correct_dates = concate_data.sort_index(ascending = True) + +concate_data.to_excel(output_file, index=True, header=True) + + +''' +#This part of the codecode is for daily only + + +merged_dfs=[] + +for df in df2: + merged_df = list_key.merge(df, how= "inner", left_on ='date', right_on = 'Date') + merged_dfs.append(merged_df) + + +final_merge = pd.concat(merged_dfs, axis= 1) +final_merge.to_excel(output_file, index=True, header=True) +print(final_merge) +'''