Warsaw Weather Analysis

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt
import seaborn as sns

# Load the data
data = pd.read_csv(r'C:\Users\Francesco\Desktop\Progetti Portfolio/warsaw.csv')

data.head(10)

# Convert the 'DATE' column to datetime format
data['DATE'] = pd.to_datetime(data['DATE'])

# Extract year, month, and day from the 'DATE' column
data['YEAR'] = data['DATE'].dt.year
data['MONTH'] = data['DATE'].dt.month
data['DAY'] = data['DATE'].dt.day

data.head(30)

# Impute missing values using the median of the same month in the given year
for column in ['PRCP', 'SNWD', 'TAVG', 'TMAX', 'TMIN']:
    data[column] = data.groupby(['YEAR', 'MONTH'])[column].transform(lambda x: x.fillna(x.median()))

# Check if there are still missing values
missing_values = data.isnull().sum()
print(missing_values)

# Filter rows for months from May to September
summer_data = data[(data['MONTH'] >= 6) & (data['MONTH'] <= 9)]

# Display the data
print(summer_data)

# Impute remaining missing values in 'SNWD' with 0
data['SNWD'] = data['SNWD'].fillna(0)

# Check if there are still missing values
missing_values = data.isnull().sum()
print(missing_values)

# Filter rows for months from May to September
summer_data = data[(data['MONTH'] >= 6) & (data['MONTH'] <= 9)]

# Display the data
print(summer_data)

# Descriptive statistics for each relevant variable
desc_stats = data[['PRCP', 'SNWD', 'TAVG', 'TMAX', 'TMIN']].describe()

print(desc_stats)

# Outlier Detection: Z-score method
from scipy import stats
# Calculate the number of outliers for each variable
num_outliers = {}
for column in ['PRCP', 'SNWD', 'TAVG', 'TMAX', 'TMIN']:
    z_scores = stats.zscore(data[column])
    abs_z_scores = np.abs(z_scores)
    outliers = abs_z_scores > 3  # a common threshold for outliers is a z-score greater than 3
    num_outliers[column] = outliers.sum()

# Print the number of outliers for each variable
for variable, num in num_outliers.items():
    print(f'{variable}: {num} outliers')

# Boxplots for temperature variables
f, axes = plt.subplots(1, 3, figsize=(15, 5))
sns.boxplot(data=data['TAVG'], color="skyblue", ax=axes[0])
sns.boxplot(data=data['TMAX'], color="teal", ax=axes[1])
sns.boxplot(data=data['TMIN'], color="pink", ax=axes[2])
axes[0].set_title('Average Temperature')
axes[1].set_title('Max Temperature')
axes[2].set_title('Min Temperature')
plt.tight_layout()
plt.show()

# Group the data by year and calculate the mean temperature for each year
data_yearly = data.groupby('YEAR').mean().reset_index()

# Define the predictors (features) and the target
features_yearly = ['YEAR']
target_yearly = ['TAVG']

# Split the data into training and testing sets
X_train_yearly, X_test_yearly, y_train_yearly, y_test_yearly = train_test_split(
    data_yearly[features_yearly], data_yearly[target_yearly], test_size=0.2, random_state=42)

# Train the model
model_yearly = LinearRegression()
model_yearly.fit(X_train_yearly, y_train_yearly)

# Make predictions using the testing set
y_pred_yearly = model_yearly.predict(X_test_yearly)

# Evaluate the model
rmse_yearly = np.sqrt(mean_squared_error(y_test_yearly, y_pred_yearly))
r2_yearly = r2_score(y_test_yearly, y_pred_yearly)

# Print the RMSE and R2 score
print(f'Root Mean Squared Error (RMSE): {rmse_yearly}')
print(f'R-squared (R2) Score: {r2_yearly}')

# Plot the actual vs predicted average temperatures
plt.figure(figsize=(10, 6))
plt.scatter(X_test_yearly, y_test_yearly, color='blue', label='Actual')
plt.plot(X_test_yearly, y_pred_yearly, color='red', linewidth=2, label='Predicted')
plt.title('Actual vs Predicted Average Temperatures')
plt.xlabel('Year')
plt.ylabel('Average Temperature (°C)')
plt.legend()
plt.grid(True)
plt.show()

# Group the data by month and calculate the mean temperature for each month
data_monthly = data.groupby('MONTH').mean().reset_index()

# Create a line plot to visualize the average temperature over the months
plt.figure(figsize=(10, 6))
sns.lineplot(x='MONTH', y='TAVG', data=data_monthly)
plt.title('Average Temperature by Month')
plt.xlabel('Month')
plt.ylabel('Average Temperature (°C)')
plt.xticks(ticks=range(1, 13), labels=['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])
plt.grid(True)
plt.show()

# Heatmap of monthly average temperatures
avg_temp_monthly = data.pivot_table(values='TAVG', index='MONTH', columns='YEAR').round(0)
plt.figure(figsize=(15, 10))
sns.heatmap(avg_temp_monthly, cmap='coolwarm', annot=True, fmt=".0f", annot_kws={'size': 10})
plt.title('Monthly Average Temperature Across Years')
plt.xlabel('Year')
plt.ylabel('Month')
plt.show()

# Calculate the 90th percentile temperature
temp_90th_percentile = data['TAVG'].quantile(0.9)

# Calculate the number of extremely hot days each year
hot_days = data[data['TAVG'] > temp_90th_percentile]
hot_days_yearly = hot_days.groupby('YEAR').size()

# Bar plot of the number of extremely hot days each year with trendline
plt.figure(figsize=(15, 5))
plt.bar(hot_days_yearly.index, hot_days_yearly.values, color='red')
sns.regplot(x=hot_days_yearly.index, y=hot_days_yearly.values, scatter=False, color='black')
plt.title('Number of Extremely Hot Days Each Year')
plt.ylabel('Number of Days')
plt.xlabel('Year')
plt.show()

# Define "extreme" events as those that fall in the top 1% of temperatures and precipitation
extreme_low_temp = data['TAVG'].quantile(0.01)
extreme_precip = data['PRCP'].quantile(0.99)

# Create a new dataframe to store the number of extreme events for each year
extreme_events = data.groupby('YEAR').apply(lambda x: pd.Series({
    'LowTemps': (x['TAVG'] < extreme_low_temp).sum(),
    'HighPrecip': (x['PRCP'] > extreme_precip).sum()})).reset_index()

# Visualize the number of extreme events over time
plt.subplot(312)
sns.lineplot(x='YEAR', y='LowTemps', data=extreme_events, color='blue')
plt.title('Number of Extremely Cold Days Over Time')
plt.ylabel('Number of Days')
plt.grid(True)
plt.subplot(313)
sns.lineplot(x='YEAR', y='HighPrecip', data=extreme_events, color='green')
plt.title('Number of Heavy Rainfall Days Over Time')
plt.ylabel('Number of Days')
plt.grid(True)
plt.tight_layout()

---

data.to_excel(r'C:\Users\Francesco\Desktop\Progetti Portfolio\Python_WarsawWeather/warsaw_filled.xlsx', index=False)

1. Overview:
   The “Dashboard_WarsawWeather.xlsx” Excel workbook delivers a comprehensive visualization and analysis of Warsaw’s daily weather data, spanning from 1993 to 2022.
   This file is an essential tool for understanding weather patterns, trends, and significant events in Warsaw over the years.
2. Data Source:
   The weather data integrated into this dashboard originates from a dataset publicly accessible on Kaggle (Link). The dataset encompasses daily metrics such as average, maximum, and minimum temperatures, precipitation levels, and snow depths.
3. Workbook Sheet Structure:
   For user convenience and easy navigation, the workbook is organized as follows:
   Data: Contains the raw data and the columns created for the project.
   Cards: Dedicated to generating the cards and the Timeline Slicer for the Detailed Time Analysis.
   Seasonality: Crafts the monthly temperature graph to illustrate seasonality.
   Probability: Analyzes precipitation and snow trends in the Seasonality Analysis.
   Timeline: Dedicated to the yearly trend analysis graphs.
   Colours: Displays the color palette employed in the dashboard.
4. How to Use:
   Dashboard Viewing: Navigate to the “Dashboard” sheet to access the curated visual insights.
   Seasonality Analysis: Data here is categorized by month. The “Probability of Day with Snow per Month” graph represents the likelihood of encountering snow in Warsaw, not necessarily the probability of snowfall on any given day.
   Detailed Time Analysis: Use the Timeline Slicer to pinpoint specific daily/monthly/quarterly/yearly values from 01/01/1993 to 31/12/2022.
   Yearly Trend Analysis: This section groups data by year for a lucid understanding of evolving trends.
   If you wish to inspect or modify the raw data, head to the “Data” sheet. Please be cautious when altering data to maintain dashboard accuracy.
5. Data Cleaning and Preprocessing:
   The data was explored and manipulate with Python.
6. Notes & Considerations:
   The “warsaw_filled.xlsx” dashboard aims to provide a comprehensive view of Warsaw’s weather patterns, but it’s essential to approach the data with an analytical mindset and consider potential external factors that might affect the readings.
   The workbook is optimized for use in the latest versions of Microsoft Excel. Display or functionality might vary slightly in older versions or different spreadsheet software.
7. Feedback & Suggestions:
   Your feedback is invaluable. If you have insights, corrections, or any queries regarding the data and its presentation, please refer back to this README or contact the creator.
8. Credits & Acknowledgments:
   A big thank you to the original data providers on Kaggle for making this dataset available to the public.