#!/usr/bin/env python3 """ Statistical Analysis Script Generated by SmartStat Agent Query: perform some descriptive statistics and some summary plotting of the data in this dataset. Generated: 2026-02-10T14:41:48.058888 """ import pandas as pd import numpy as np import matplotlib.pyplot as plt import seaborn as sns from scipy import stats import warnings warnings.filterwarnings('ignore') def main(): print("Starting statistical analysis...") print(f"Query: perform some descriptive statistics and some summary plotting of the data in this dataset.") # Load data try: df = pd.read_csv('input_data.csv') print(f"Data loaded successfully: {df.shape}") except Exception as e: print(f"Error loading data: {e}") return # Initialize conclusions file conclusions = [] conclusions.append("=" * 80) conclusions.append("DESCRIPTIVE STATISTICS AND SUMMARY ANALYSIS") conclusions.append("=" * 80) conclusions.append(f"\nDataset Shape: {df.shape[0]} rows × {df.shape[1]} columns\n") # ======================================================================== # 1. DATA OVERVIEW AND QUALITY ASSESSMENT # ======================================================================== print("\n1. Analyzing data quality and structure...") # Create data quality summary quality_summary = [] for col in df.columns: null_count = df[col].isnull().sum() null_pct = (null_count / len(df)) * 100 dtype = df[col].dtype unique_count = df[col].nunique() quality_summary.append({ 'Column': col, 'Data_Type': str(dtype), 'Non_Null_Count': len(df) - null_count, 'Null_Count': null_count, 'Null_Percentage': round(null_pct, 2), 'Unique_Values': unique_count }) quality_df = pd.DataFrame(quality_summary) quality_df.to_csv('table_01_data_quality_summary.csv', index=False) print(" Saved: table_01_data_quality_summary.csv") conclusions.append("DATA QUALITY SUMMARY") conclusions.append("-" * 80) conclusions.append(f"Total columns: {len(df.columns)}") conclusions.append(f"Columns with >50% missing data: {len(quality_df[quality_df['Null_Percentage'] > 50])}") conclusions.append(f"Columns with >90% missing data: {len(quality_df[quality_df['Null_Percentage'] > 90])}") high_missing = quality_df[quality_df['Null_Percentage'] > 50]['Column'].tolist() if high_missing: conclusions.append(f"\nColumns with high missing data (>50%): {', '.join(high_missing)}") # ======================================================================== # 2. IDENTIFY AND ANALYZE NUMERIC COLUMNS # ======================================================================== print("\n2. Analyzing numeric columns...") # Try to convert object columns to numeric where possible numeric_cols = [] for col in df.columns: if df[col].dtype in ['int64', 'float64']: numeric_cols.append(col) elif df[col].dtype == 'object': # Try to convert to numeric try: converted = pd.to_numeric(df[col], errors='coerce') if converted.notna().sum() > len(df) * 0.1: # At least 10% valid numeric values df[col + '_numeric'] = converted numeric_cols.append(col + '_numeric') except: pass if numeric_cols: # Descriptive statistics for numeric columns numeric_df = df[numeric_cols].select_dtypes(include=[np.number]) if not numeric_df.empty: desc_stats = numeric_df.describe().T desc_stats['missing'] = numeric_df.isnull().sum() desc_stats['missing_pct'] = (desc_stats['missing'] / len(df)) * 100 desc_stats['skewness'] = numeric_df.skew() desc_stats['kurtosis'] = numeric_df.kurtosis() desc_stats.to_csv('table_02_numeric_descriptive_statistics.csv') print(" Saved: table_02_numeric_descriptive_statistics.csv") conclusions.append("\n\nNUMERIC VARIABLES ANALYSIS") conclusions.append("-" * 80) conclusions.append(f"Number of numeric columns analyzed: {len(numeric_df.columns)}") for col in numeric_df.columns: if numeric_df[col].notna().sum() > 0: conclusions.append(f"\n{col}:") conclusions.append(f" Mean: {numeric_df[col].mean():.2f}") conclusions.append(f" Median: {numeric_df[col].median():.2f}") conclusions.append(f" Std Dev: {numeric_df[col].std():.2f}") conclusions.append(f" Range: [{numeric_df[col].min():.2f}, {numeric_df[col].max():.2f}]") conclusions.append(f" Missing: {numeric_df[col].isnull().sum()} ({(numeric_df[col].isnull().sum()/len(df)*100):.1f}%)") # ======================================================================== # 3. ANALYZE CATEGORICAL COLUMNS # ======================================================================== print("\n3. Analyzing categorical columns...") categorical_cols = df.select_dtypes(include=['object']).columns.tolist() # Filter out columns with too many missing values or too many unique values useful_categorical = [] for col in categorical_cols: null_pct = (df[col].isnull().sum() / len(df)) * 100 unique_count = df[col].nunique() if null_pct < 90 and 1 < unique_count < 100: # Reasonable categorical variables useful_categorical.append(col) if useful_categorical: categorical_summary = [] for col in useful_categorical[:10]: # Limit to first 10 for manageability value_counts = df[col].value_counts() top_5 = value_counts.head(5) for idx, (value, count) in enumerate(top_5.items()): categorical_summary.append({ 'Column': col, 'Rank': idx + 1, 'Value': str(value), 'Count': count, 'Percentage': round((count / len(df)) * 100, 2) }) cat_summary_df = pd.DataFrame(categorical_summary) cat_summary_df.to_csv('table_03_categorical_frequency_summary.csv', index=False) print(" Saved: table_03_categorical_frequency_summary.csv") conclusions.append("\n\nCATEGORICAL VARIABLES ANALYSIS") conclusions.append("-" * 80) conclusions.append(f"Number of categorical columns analyzed: {len(useful_categorical)}") for col in useful_categorical[:5]: # Top 5 for conclusions top_value = df[col].value_counts().index[0] if len(df[col].value_counts()) > 0 else "N/A" top_count = df[col].value_counts().iloc[0] if len(df[col].value_counts()) > 0 else 0 conclusions.append(f"\n{col}:") conclusions.append(f" Unique values: {df[col].nunique()}") conclusions.append(f" Most frequent: '{top_value}' ({top_count} occurrences)") conclusions.append(f" Missing: {df[col].isnull().sum()} ({(df[col].isnull().sum()/len(df)*100):.1f}%)") # ======================================================================== # 4. VISUALIZATIONS # ======================================================================== print("\n4. Creating visualizations...") # Plot 1: Missing Data Heatmap try: plt.figure(figsize=(12, 8)) missing_data = df.isnull() # Only show columns with some missing data cols_with_missing = [col for col in df.columns if df[col].isnull().sum() > 0] if cols_with_missing: sns.heatmap(missing_data[cols_with_missing].head(100), cbar=True, yticklabels=False, cmap='viridis') plt.title('Missing Data Pattern (First 100 Rows)', fontsize=14, fontweight='bold') plt.xlabel('Columns', fontsize=12) plt.ylabel('Rows', fontsize=12) plt.xticks(rotation=45, ha='right') plt.tight_layout() plt.savefig('plot_01_missing_data_heatmap.png', dpi=300, bbox_inches='tight') plt.close() print(" Saved: plot_01_missing_data_heatmap.png") except Exception as e: print(f" Warning: Could not create missing data heatmap: {e}") # Plot 2: Missing Data Bar Chart try: plt.figure(figsize=(12, 6)) missing_pct = (df.isnull().sum() / len(df) * 100).sort_values(ascending=False) missing_pct = missing_pct[missing_pct > 0] if len(missing_pct) > 0: colors = ['red' if x > 50 else 'orange' if x > 20 else 'yellow' for x in missing_pct] missing_pct.plot(kind='bar', color=colors) plt.title('Percentage of Missing Data by Column', fontsize=14, fontweight='bold') plt.xlabel('Columns', fontsize=12) plt.ylabel('Missing Data (%)', fontsize=12) plt.xticks(rotation=45, ha='right') plt.axhline(y=50, color='r', linestyle='--', alpha=0.5, label='50% threshold') plt.legend() plt.tight_layout() plt.savefig('plot_02_missing_data_percentage.png', dpi=300, bbox_inches='tight') plt.close() print(" Saved: plot_02_missing_data_percentage.png") except Exception as e: print(f" Warning: Could not create missing data bar chart: {e}") # Plot 3: Numeric Variables Distribution if numeric_cols: try: numeric_df_clean = df[numeric_cols].select_dtypes(include=[np.number]) # Remove columns with all NaN numeric_df_clean = numeric_df_clean.dropna(axis=1, how='all') if not numeric_df_clean.empty and len(numeric_df_clean.columns) > 0: n_cols = min(len(numeric_df_clean.columns), 6) # Max 6 plots fig, axes = plt.subplots(2, 3, figsize=(15, 10)) axes = axes.flatten() for idx, col in enumerate(numeric_df_clean.columns[:n_cols]): data = numeric_df_clean[col].dropna() if len(data) > 0: axes[idx].hist(data, bins=30, edgecolor='black', alpha=0.7, color='skyblue') axes[idx].set_title(f'{col}\n(n={len(data)})', fontsize=10) axes[idx].set_xlabel('Value', fontsize=9) axes[idx].set_ylabel('Frequency', fontsize=9) axes[idx].grid(alpha=0.3) # Hide unused subplots for idx in range(n_cols, 6): axes[idx].axis('off') plt.suptitle('Distribution of Numeric Variables', fontsize=14, fontweight='bold') plt.tight_layout() plt.savefig('plot_03_numeric_distributions.png', dpi=300, bbox_inches='tight') plt.close() print(" Saved: plot_03_numeric_distributions.png") except Exception as e: print(f" Warning: Could not create numeric distributions: {e}") # Plot 4: Categorical Variables Bar Charts if useful_categorical: try: n_plots = min(len(useful_categorical), 6) fig, axes = plt.subplots(2, 3, figsize=(15, 10)) axes = axes.flatten() for idx, col in enumerate(useful_categorical[:n_plots]): value_counts = df[col].value_counts().head(10) axes[idx].barh(range(len(value_counts)), value_counts.values, color='coral') axes[idx].set_yticks(range(len(value_counts))) axes[idx].set_yticklabels([str(x)[:20] for x in value_counts.index], fontsize=8) axes[idx].set_xlabel('Count', fontsize=9) axes[idx].set_title(f'{col}\n(Top 10 values)', fontsize=10) axes[idx].grid(axis='x', alpha=0.3) # Hide unused subplots for idx in range(n_plots, 6): axes[idx].axis('off') plt.suptitle('Frequency of Categorical Variables', fontsize=14, fontweight='bold') plt.tight_layout() plt.savefig('plot_04_categorical_frequencies.png', dpi=300, bbox_inches='tight') plt.close() print(" Saved: plot_04_categorical_frequencies.png") except Exception as e: print(f" Warning: Could not create categorical frequencies: {e}") # Plot 5: Correlation Heatmap (if numeric columns exist) if numeric_cols: try: numeric_df_clean = df[numeric_cols].select_dtypes(include=[np.number]) numeric_df_clean = numeric_df_clean.dropna(axis=1, how='all') if not numeric_df_clean.empty and len(numeric_df_clean.columns) > 1: # Calculate correlation only for columns with sufficient data corr_df = numeric_df_clean.loc[:, numeric_df_clean.notna().sum() > len(df) * 0.1] if len(corr_df.columns) > 1: correlation_matrix = corr_df.corr() plt.figure(figsize=(10, 8)) sns.heatmap(correlation_matrix, annot=True, fmt='.2f', cmap='coolwarm', center=0, square=True, linewidths=1, cbar_kws={"shrink": 0.8}) plt.title('Correlation Matrix of Numeric Variables', fontsize=14, fontweight='bold') plt.tight_layout() plt.savefig('plot_05_correlation_heatmap.png', dpi=300, bbox_inches='tight') plt.close() print(" Saved: plot_05_correlation_heatmap.png") # Save correlation matrix as table correlation_matrix.to_csv('table_04_correlation_matrix.csv') print(" Saved: table_04_correlation_matrix.csv") except Exception as e: print(f" Warning: Could not create correlation heatmap: {e}") # Plot 6: Data Completeness Overview try: plt.figure(figsize=(10, 6)) completeness = ((df.notna().sum() / len(df)) * 100).sort_values(ascending=True) colors = ['green' if x > 80 else 'orange' if x > 50 else 'red' for x in completeness] completeness.plot(kind='barh', color=colors) plt.title('Data Completeness by Column', fontsize=14, fontweight='bold') plt.xlabel('Completeness (%)', fontsize=12) plt.ylabel('Columns', fontsize=12) plt.axvline(x=80, color='g', linestyle='--', alpha=0.5, label='80% threshold') plt.axvline(x=50, color='orange', linestyle='--', alpha=0.5, label='50% threshold') plt.legend() plt.tight_layout() plt.savefig('plot_06_data_completeness.png', dpi=300, bbox_inches='tight') plt.close() print(" Saved: plot_06_data_completeness.png") except Exception as e: print(f" Warning: Could not create data completeness plot: {e}") # ======================================================================== # 5. ADDITIONAL SUMMARY STATISTICS # ======================================================================== print("\n5. Generating additional summary statistics...") # Overall dataset summary overall_summary = { 'Metric': [ 'Total Rows', 'Total Columns', 'Numeric Columns', 'Categorical Columns', 'Total Missing Values', 'Missing Data Percentage', 'Complete Rows', 'Complete Rows Percentage' ], 'Value': [ df.shape[0], df.shape[1], len(numeric_cols), len(categorical_cols), df.isnull().sum().sum(), round((df.isnull().sum().sum() / (df.shape[0] * df.shape[1])) * 100, 2), df.dropna().shape[0], round((df.dropna().shape[0] / df.shape[0]) * 100, 2) ] } overall_df = pd.DataFrame(overall_summary) overall_df.to_csv('table_05_overall_dataset_summary.csv', index=False) print(" Saved: table_05_overall_dataset_summary.csv") # ======================================================================== # 6. WRITE CONCLUSIONS # ======================================================================== conclusions.append("\n\nOVERALL DATASET SUMMARY") conclusions.append("-" * 80) conclusions.append(f"Total observations: {df.shape[0]}") conclusions.append(f"Total variables: {df.shape[1]}") conclusions.append(f"Numeric variables: {len(numeric_cols)}") conclusions.append(f"Categorical variables: {len(categorical_cols)}") conclusions.append(f"\nData completeness: {round((df.dropna().shape[0] / df.shape[0]) * 100, 2)}% complete rows") conclusions.append(f"Total missing values: {df.isnull().sum().sum()} ({round((df.isnull().sum().sum() / (df.shape[0] * df.shape[1])) * 100, 2)}%)") conclusions.append("\n\nKEY FINDINGS") conclusions.append("-" * 80) # Identify columns with most missing data high_missing_cols = quality_df[quality_df['Null_Percentage'] > 90].sort_values('Null_Percentage', ascending=False) if len(high_missing_cols) > 0: conclusions.append(f"\n1. High Missing Data Alert:") conclusions.append(f" {len(high_missing_cols)} columns have >90% missing data.") conclusions.append(f" These columns may not be useful for analysis.") # Identify most complete columns complete_cols = quality_df[quality_df['Null_Percentage'] < 10].sort_values('Null_Percentage') if len(complete_cols) > 0: conclusions.append(f"\n2. Most Complete Columns:") conclusions.append(f" {len(complete_cols)} columns have <10% missing data.") conclusions.append(f" These are the most reliable for analysis.") # Data type distribution conclusions.append(f"\n3. Data Type Distribution:") conclusions.append(f" Object/String columns: {len(df.select_dtypes(include=['object']).columns)}") conclusions.append(f" Numeric columns: {len(df.select_dtypes(include=[np.number]).columns)}") conclusions.append("\n\nRECOMMENDATIONS") conclusions.append("-" * 80) conclusions.append("1. Consider removing columns with >90% missing data") conclusions.append("2. Investigate the cause of missing data patterns") conclusions.append("3. Focus analysis on columns with <20% missing data") conclusions.append("4. Consider data imputation strategies for important variables") conclusions.append("5. Verify data types and convert where appropriate") conclusions.append("\n" + "=" * 80) conclusions.append("END OF ANALYSIS") conclusions.append("=" * 80) # Write conclusions to file with open('conclusions.txt', 'w') as f: f.write('\n'.join(conclusions)) print("\n Saved: conclusions.txt") print("\n" + "=" * 80) print("Analysis completed successfully!") print("=" * 80) print("\nGenerated files:") print(" - table_01_data_quality_summary.csv") if numeric_cols: print(" - table_02_numeric_descriptive_statistics.csv") if useful_categorical: print(" - table_03_categorical_frequency_summary.csv") if numeric_cols and len(numeric_cols) > 1: print(" - table_04_correlation_matrix.csv") print(" - table_05_overall_dataset_summary.csv") print(" - plot_01_missing_data_heatmap.png") print(" - plot_02_missing_data_percentage.png") if numeric_cols: print(" - plot_03_numeric_distributions.png") if useful_categorical: print(" - plot_04_categorical_frequencies.png") if numeric_cols and len(numeric_cols) > 1: print(" - plot_05_correlation_heatmap.png") print(" - plot_06_data_completeness.png") print(" - conclusions.txt") print("\n" + "=" * 80) if __name__ == "__main__": main()