pythonadvanced

Flyte ML Pipeline in Python

Define a reproducible machine learning workflow with Flyte's Python SDK for data-to-model pipelines.

python
from flytekit import task, workflow, Resources
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
import pandas as pd
from typing import Tuple

@task(requests=Resources(cpu='1', mem='500Mi'))
def load_data() -> Tuple[pd.DataFrame, pd.Series]:
    X, y = load_iris(return_X_y=True, as_frame=True)
    return X, y

@task
def split_data(X: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series]:
    return train_test_split(X, y, test_size=0.2, random_state=42)

@task
def train_model(X_train: pd.DataFrame, y_train: pd.Series) -> RandomForestClassifier:
    model = RandomForestClassifier(n_estimators=100, random_state=42)
    model.fit(X_train, y_train)
    return model

@task
def evaluate(model: RandomForestClassifier, X_test: pd.DataFrame, y_test: pd.Series) -> float:
    return accuracy_score(y_test, model.predict(X_test))

@workflow
def ml_pipeline() -> float:
    X, y = load_data()
    X_train, X_test, y_train, y_test = split_data(X=X, y=y)
    model = train_model(X_train=X_train, y_train=y_train)
    return evaluate(model=model, X_test=X_test, y_test=y_test)

Use Cases

  • ML orchestration
  • reproducible pipelines
  • MLOps

Tags

Related Snippets

Similar patterns you can reuse in the same workflow.