Data Science — Cours Complet

1. Pandas — Manipulation de Données

Structure de Base

import pandas as pd
import numpy as np

# Series — 1D
s = pd.Series([1, 2, 3, 4, 5], name="numbers")

# DataFrame — 2D
df = pd.DataFrame({
    "name": ["Alice", "Bob", "Charlie"],
    "age": [25, 30, 35],
    "salary": [50000, 60000, 70000],
})

# Lecture/Écriture
df_csv = pd.read_csv("data.csv")
df_parquet = pd.read_parquet("data.parquet")
df.to_csv("output.csv", index=False)
df.to_parquet("output.parquet")

Exploration

# Aperçu
df.head(10)
df.tail(5)
df.info()
df.describe()
df.shape
df.columns
df.dtypes

# Statistiques
df.describe(include="all")
df["age"].mean()
df["age"].median()
df["age"].std()
df["age"].quantile([0.25, 0.5, 0.75])

Selection et Filtrage

# Selection de colonnes
df["name"]
df[["name", "age"]]

# Selection de lignes
df.iloc[0]          # première ligne (index)
df.iloc[1:3]        # lignes 1-2
df.loc[0]           # par index label
df.loc[df["age"] > 30]  # condition

# Filtrage avancé
young_high_earners = df[
    (df["age"] < 30) & (df["salary"] > 55000)
]

# Query string
df.query("age > 25 and salary > 50000")

GroupBy

# GroupBy basique
df.groupby("category")["sales"].sum()

# Agrégations multiples
df.groupby("category").agg({
    "sales": ["sum", "mean", "count"],
    "profit": "sum",
})

# GroupBy avec transformations
df["sales_rank"] = df.groupby("region")["sales"].rank(ascending=False)

# GroupBy apply personnalisé
df.groupby("category").apply(
    lambda x: x.sort_values("sales", ascending=False).head(3)
)

Merge et Join

# Merge (SQL-style)
orders = pd.DataFrame({
    "order_id": [1, 2, 3],
    "customer_id": [101, 102, 101],
    "amount": [100, 200, 150],
})
customers = pd.DataFrame({
    "customer_id": [101, 102],
    "name": ["Alice", "Bob"],
})

# Inner join
result = orders.merge(customers, on="customer_id", how="inner")

# Left join
result = orders.merge(customers, on="customer_id", how="left")

# Concatenation
pd.concat([df1, df2], axis=0)  # rows
pd.concat([df1, df2], axis=1)  # columns

Pivot Tables

# Pivot
pivot = df.pivot_table(
    values="sales",
    index="region",
    columns="product",
    aggfunc="sum",
    fill_value=0,
)

# Melt (inverse de pivot)
melted = pd.melt(
    df,
    id_vars=["region"],
    value_vars=["product_a", "product_b"],
    var_name="product",
    value_name="sales",
)

Apply et Vectorization

# apply — mais préférer la vectorization
df["name_length"] = df["name"].apply(len)

# Vectorization (100-1000x plus rapide)
df["salary_raised"] = df["salary"] * 1.1
df["is_adult"] = df["age"] >= 18
df["full_name"] = df["first_name"] + " " + df["last_name"]

# map
df["gender_code"] = df["gender"].map({"Male": 0, "Female": 1})

# apply avec axis=1 (lent — à éviter sur grands datasets)
df["score"] = df.apply(
    lambda row: row["math"] + row["science"], axis=1
)

2. NumPy — Calcul Numérique

Arrays

import numpy as np

# Création
arr = np.array([1, 2, 3, 4, 5])
zeros = np.zeros((3, 4))
ones = np.ones((2, 3))
eye = np.eye(5)  # identité
random = np.random.randn(1000)

# Propriétés
arr.shape
arr.dtype
arr.ndim
arr.size
arr.nbytes  # mémoire

Broadcasting

# Broadcasting — opérations sur des shapes différents
a = np.array([[1, 2, 3], [4, 5, 6]])  # (2, 3)
b = np.array([10, 20, 30])            # (3,)
c = a + b                              # (2, 3) via broadcasting

# Règles du broadcasting
# 1. Aligner les dimensions par la droite
# 2. Les dimensions manquantes ou de taille 1 sont "broadcastées"

Algèbre Linéaire

A = np.array([[1, 2], [3, 4]])
B = np.array([[5, 6], [7, 8]])

# Multiplication matricielle
C = A @ B
C = np.matmul(A, B)
C = A.dot(B)

# Produit élément par élément
D = A * B

# Transposition
A.T

# Inverse
np.linalg.inv(A)

# Déterminant
np.linalg.det(A)

# Valeurs propres
eigvals, eigvecs = np.linalg.eig(A)

# SVD
U, S, Vt = np.linalg.svd(A)

3. Polars — DataFrames Haute Performance

Eager vs Lazy

import polars as pl

# Eager API (similaire Pandas)
df = pl.DataFrame({
    "name": ["Alice", "Bob", "Charlie"],
    "age": [25, 30, 35],
    "salary": [50000, 60000, 70000],
})

# Lazy API — optimisé
q = (
    pl.scan_csv("large_file.csv")
    .filter(pl.col("age") > 25)
    .group_by("department")
    .agg(pl.col("salary").mean())
    .sort("salary", descending=True)
)
df_result = q.collect()  # exécution paresseuse

Expression API

# API Expression (différente de Pandas)
df = pl.DataFrame({
    "name": ["Alice", "Bob", "Charlie"],
    "age": [25, 30, 35],
    "salary": [50000, 60000, 70000],
})

df.with_columns(
    (pl.col("salary") * 1.1).alias("salary_raised"),
    pl.when(pl.col("age") >= 30)
        .then(pl.lit("senior"))
        .otherwise(pl.lit("junior"))
        .alias("level"),
)

df.group_by("level").agg([
    pl.col("salary").mean().alias("avg_salary"),
    pl.col("age").count().alias("count"),
])

4. Visualisation

Matplotlib

import matplotlib.pyplot as plt

# Style
plt.style.use("seaborn-v0_8")

# Line plot
plt.figure(figsize=(10, 6))
plt.plot(x, y, label="Series A")
plt.plot(x, z, label="Series B")
plt.xlabel("X axis")
plt.ylabel("Y axis")
plt.title("Title")
plt.legend()
plt.grid(True)
plt.show()

# Subplots
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
axes[0, 0].plot(x, y)
axes[0, 1].scatter(x, y)
axes[1, 0].hist(y, bins=30)
axes[1, 1].boxplot([y, z])

Seaborn

import seaborn as sns

# Distribution
sns.histplot(data=df, x="age", hue="gender")
sns.kdeplot(data=df, x="salary", fill=True)

# Relations
sns.scatterplot(data=df, x="age", y="salary", hue="department")
sns.pairplot(df, hue="category")
sns.heatmap(df.corr(), annot=True, cmap="coolwarm")

# Catégories
sns.boxplot(data=df, x="department", y="salary")
sns.violinplot(data=df, x="department", y="salary")
sns.barplot(data=df, x="department", y="salary")

Plotly — Interactif

import plotly.express as px
import plotly.graph_objects as go

# Express (haut niveau)
fig = px.scatter(df, x="age", y="salary", color="department",
                 size="experience", hover_data=["name"])
fig.show()

# Graph Objects (bas niveau)
fig = go.Figure()
fig.add_trace(go.Scatter(x=x, y=y, mode="lines+markers", name="Series"))
fig.update_layout(title="Interactive Plot", xaxis_title="X", yaxis_title="Y")
fig.show()

5. Jupyter Écosystème

Jupyter Lab

pip install jupyterlab voila
jupyter lab

Widgets interactifs

import ipywidgets as widgets
from IPython.display import display

@widgets.interact(x=(0, 10, 0.1), y=(0, 10, 0.1))
def plot_surface(x=5.0, y=5.0):
    plt.figure(figsize=(8, 6))
    plt.scatter([x], [y], s=200, c="red")
    plt.xlim(0, 10)
    plt.ylim(0, 10)
    plt.grid(True)
    plt.show()

Voila — Dashboard

voila notebook.ipynb  # transforme en application web

6. Performance — Vectorization vs Loops

import time

size = 10_000_000
data = np.random.randn(size)

# Loop Python (lent)
start = time.perf_counter()
result = []
for x in data:
    result.append(x * 2 + 1)
print(f"Loop: {time.perf_counter() - start:.2f}s")

# List comprehension
start = time.perf_counter()
result = [x * 2 + 1 for x in data]
print(f"List comp: {time.perf_counter() - start:.2f}s")

# NumPy vectorization (rapide)
start = time.perf_counter()
result = data * 2 + 1
print(f"Vectorized: {time.perf_counter() - start:.2f}s")

7. ETL Pipelines

"""Pipeline ETL typique."""

from typing import Generator
import pandas as pd
import logging

logger = logging.getLogger(__name__)

def extract(filepath: str, chunk_size: int = 10000) -> Generator[pd.DataFrame, None, None]:
    """Extract — lecture par chunks."""
    for chunk in pd.read_csv(filepath, chunksize=chunk_size):
        logger.info(f"Extracted {len(chunk)} rows")
        yield chunk

def transform(df: pd.DataFrame) -> pd.DataFrame:
    """Transform — nettoyage et enrichissement."""
    df = df.dropna(subset=["id", "value"])
    df["value"] = pd.to_numeric(df["value"], errors="coerce")
    df["date"] = pd.to_datetime(df["date"])
    df["year_month"] = df["date"].dt.to_period("M")
    return df

def load(df: pd.DataFrame, output_path: str, mode: str = "a") -> None:
    """Load — écriture parquet."""
    df.to_parquet(
        output_path,
        engine="pyarrow",
        compression="snappy",
        append=(mode == "a"),
    )

def run_pipeline(input_path: str, output_path: str) -> None:
    """Exécute le pipeline ETL complet."""
    for i, chunk in enumerate(extract(input_path)):
        transformed = transform(chunk)
        mode = "w" if i == 0 else "a"
        load(transformed, output_path, mode)
        logger.info(f"Processed chunk {i}: {len(transformed)} rows")