import requests
import pandas as pd
from datetime import datetime, timedelta

TB_HOST    = 'https://thingsboard.cloud'
TB_EMAIL   = 'your@email.com'
TB_PASSWORD= 'yourPassword'
DEVICE_ID  = 'YOUR_DEVICE_ID'

def get_token():
    r = requests.post(f'{TB_HOST}/api/auth/login',
                      json={'username': TB_EMAIL, 'password': TB_PASSWORD})
    return r.json()['token']

def get_telemetry(token, keys, hours=24):
    """Retrieve last N hours of telemetry for given keys."""
    end_ms   = int(datetime.now().timestamp() * 1000)
    start_ms = int((datetime.now() - timedelta(hours=hours)).timestamp() * 1000)
    headers  = {'Authorization': f'Bearer {token}'}
    params   = {'keys': ','.join(keys), 'startTs': start_ms,
                 'endTs': end_ms, 'limit': 10000, 'agg': 'NONE'}
    url = f'{TB_HOST}/api/plugins/telemetry/DEVICE/{DEVICE_ID}/values/timeseries'
    return requests.get(url, headers=headers, params=params).json()

def to_dataframe(telemetry):
    """Convert ThingsBoard response to pandas DataFrame with datetime index."""
    dfs = []
    for key, values in telemetry.items():
        df = pd.DataFrame(values)                           # columns: ts, value
        df['ts'] = pd.to_datetime(df['ts'], unit='ms')     # epoch ms → datetime
        df = df.rename(columns={'value': key}).set_index('ts')
        df[key] = pd.to_numeric(df[key], errors='coerce')  # ensure numeric
        dfs.append(df)
    if not dfs:
        return pd.DataFrame()
    return pd.concat(dfs, axis=1).sort_index()

# Usage
token = get_token()
raw   = get_telemetry(token, ['temperature', 'humidity', 'light'], hours=48)
df    = to_dataframe(raw)
print(df.head())
print(f'Shape: {df.shape}, Date range: {df.index.min()} -- {df.index.max()}')

# Check for missing data
print(df.isnull().sum())
print(f'Missing %: {df.isnull().sum() / len(df) * 100:.1f}%')

# Strategy 1: Forward-fill (carry last known value forward)
# Good for: slowly changing signals (temperature, humidity)
df_ffill  = df.fillna(method='ffill')

# Strategy 2: Linear interpolation
# Good for: continuously varying signals with short gaps
df_interp = df.interpolate(method='time')   # time-aware interpolation

# Strategy 3: Drop rows with too many missing values
# Good for: rows with > 50% missing (don't trust partial data)
df_clean  = df.dropna(thresh=int(len(df.columns) * 0.5))

# Strategy 4: Fill with column median (robust to outliers)
df['temperature'] = df['temperature'].fillna(df['temperature'].median())

# For IoT: check if gap duration exceeds sensor reliability window
MAX_GAP   = pd.Timedelta('2h')
gaps      = df.index.to_series().diff()
large_gaps= gaps[gaps > MAX_GAP]
if not large_gaps.empty:
    print(f'WARNING: {len(large_gaps)} gaps > 2 hours detected')

# Remove exact duplicate timestamps (from MQTT retry logic)
df = df[~df.index.duplicated(keep='first')]

from scipy import stats
import numpy as np

# Z-score outlier detection (|Z| > 3 = anomaly)
def remove_zscore_outliers(series, threshold=3.0):
    z_scores = np.abs(stats.zscore(series.dropna()))
    mask     = z_scores < threshold
    cleaned  = series.copy()
    cleaned[series.index.isin(series.dropna().index[~mask])] = np.nan
    return cleaned

df['temperature_clean'] = remove_zscore_outliers(df['temperature'])

# IQR-based outlier detection (more robust to skewed distributions)
def remove_iqr_outliers(series, factor=1.5):
    Q1, Q3 = series.quantile(0.25), series.quantile(0.75)
    IQR    = Q3 - Q1
    return series.where((series >= Q1 - factor*IQR) & (series <= Q3 + factor*IQR))

df['humidity_clean'] = remove_iqr_outliers(df['humidity'])

# Domain-based outlier detection (most reliable for IoT)
# Apply physical limits from sensor datasheets
df.loc[df['temperature'] < -40, 'temperature'] = np.nan  # DHT22 min
df.loc[df['temperature'] >  80, 'temperature'] = np.nan  # DHT22 max
df.loc[df['humidity']    <   0, 'humidity']    = np.nan
df.loc[df['humidity']    > 100, 'humidity']    = np.nan

# Resample to regular 5-minute intervals
df_5min  = df.resample('5T').mean()   # 5T = 5-minute bins, mean aggregation

df_hourly = df.resample('1H').agg({
    'temperature': 'mean',   # Average temperature per hour
    'humidity':    'mean',
    'light':       'max',    # Peak light level per hour
})

# Min-Max normalisation (scale to 0–1 range)
# Required for distance-based ML algorithms (KNN, SVM)
from sklearn.preprocessing import MinMaxScaler
scaler   = MinMaxScaler()
features = ['temperature', 'humidity', 'light']
df_norm  = df_5min[features].copy()
df_norm[features] = scaler.fit_transform(df_norm[features].fillna(0))

# Standard normalisation (zero mean, unit variance)
# Required for algorithms that assume Gaussian distribution
from sklearn.preprocessing import StandardScaler
std_scaler = StandardScaler()
df_std     = df_5min[features].copy()
df_std[features] = std_scaler.fit_transform(df_std[features].fillna(0))

import pandas as pd, numpy as np

# 1. Temporal features — extract time-of-day patterns
df['hour']             = df.index.hour
df['day_of_week']      = df.index.dayofweek          # 0=Monday, 6=Sunday
df['is_weekend']       = df['day_of_week'].isin([5,6]).astype(int)
df['is_business_hours']= ((df['hour'] >= 8) & (df['hour'] < 18)).astype(int)

# 2. Rolling statistics — capture trends and variance
df['temp_rolling_mean_1h'] = df['temperature'].rolling('1h').mean()
df['temp_rolling_std_1h']  = df['temperature'].rolling('1h').std()
df['temp_rolling_max_6h']  = df['temperature'].rolling('6h').max()

# 3. Rate of change — detect rapid temperature spikes
df['temp_delta']      = df['temperature'].diff()
df['temp_delta_rate'] = df['temp_delta'] / \
    df.index.to_series().diff().dt.total_seconds() * 60   # °C per minute

# 4. Lag features — use past values as predictors
df['temp_lag_1']  = df['temperature'].shift(1)    # Previous reading
df['temp_lag_12'] = df['temperature'].shift(12)   # 1 hour ago (5-min data)

# 5. Derived features — sensor-specific calculated values
# Heat index: perceived temperature combining temp and humidity
T = df['temperature'];  H = df['humidity']
df['heat_index'] = (-8.78469475556 + 1.61139411*T + 2.33854883889*H
                    - 0.14611605*T*H  - 0.012308094*T**2
                    - 0.0164248277778*H**2 + 0.002211732*T**2*H
                    + 0.00072546*T*H**2   - 0.000003582*T**2*H**2)

print(f'Features created: {df.columns.tolist()}')