Azure Data Factory

Setup

from fabias.integrations import adf
from fabias import ServicePrincipalAuth

auth = ServicePrincipalAuth(
    tenant_id="your-tenant-id",
    client_id="your-client-id",
    client_secret="your-client-secret"
)

adf.client(
    subscription_id="your-subscription-id",
    resource_group="your-resource-group",
    factory="your-data-factory",
    auth=auth
)

Running Pipelines

from fabias.integrations import adf

# Get a pipeline
pipeline = adf.pipeline("Daily_ETL")

# Run with parameters
job = pipeline.run(parameters={
    "date": "2025-01-01",
    "mode": "full"
})

# Wait for completion
job.wait()
print(f"Status: {job.status}")

Listing Pipelines

for pipeline in adf.pipelines():
    print(f"{pipeline.name}: {pipeline.id}")

Pipeline Properties

pipeline = adf.pipeline("Daily_ETL")

print(f"Name: {pipeline.name}")
print(f"Activities: {len(pipeline.activities)}")
print(f"Parameters: {pipeline.parameters}")
print(f"Annotations: {pipeline.annotations}")

Job Tracking

job = pipeline.run()

# Poll for status
while not job.is_complete:
    job.check_status()
    print(f"Status: {job.status}")
    time.sleep(5)

# Or use wait with callback
job.wait(
    callback=lambda j: print(f"Status: {j.status}"),
    timeout=3600  # 1 hour timeout
)