Skip to main content

Datos Core Panel - Framework Setup Guide

Clickstream Core Panel - Framework Setup Guide

This guide shows you how to create frameworks for Clickstream Core Panel website traffic insights.

Clickstream Core Panel Insight IDs​

  • 379: Website Traffic
  • 380: Average Website Traffic
  • 91: Website Traffic Per User
  • 381: Website Users

Quick Start​

  1. Set up authentication
  2. Define your entities and date range
  3. Create frameworks for each insight ID
  4. Purchase and retrieve data

Step 1: Setup and Authentication​

Import required libraries and initialize the CarbonArc client:

# Import required libraries
import os
from datetime import datetime, timedelta
import pandas as pd
from carbonarc import CarbonArcClient
from dotenv import load_dotenv


# Load environment variables
load_dotenv()

# Initialize client
HOST = os.getenv("HOST", "https://api.carbonarc.co")
TOKEN = os.getenv("TOKEN")

if not TOKEN:
    raise ValueError("TOKEN not found. Please set it in your .env file.")

client = CarbonArcClient(host=HOST, token=TOKEN)
print("✅ CarbonArc client initialized")

Step 2: Define Entities and Date Range​

Define your entity sets and date range. For Clickstream Core Panel, you typically use product entities.

You have two options:

  1. Specific entities: List individual entity IDs
  2. Wildcard (*): Pull data for ALL entities of a given representation
# Option 1: Define specific entity sets (example with product entities)
entity_sets = [
    {"carc_id": 2262, "representation": "website"}, # airbnb.com
    {"carc_id": 943, "representation": "website"}, # travelocity.com
    {"carc_id": 8404, "representation": "website"}, # amextravel.com
    {"carc_id": 266, "representation": "website"}, # vrbo.com
    {"carc_id": 496, "representation": "website"}, # booking.com
    {"carc_id": 214, "representation": "website"}, # kayak.com
    {"carc_id": 835, "representation": "website"}, # tripadvisor.com
    # Add more entities as needed
]

# Option 2: Use wildcard to pull ALL entities (uncomment to use)
# entity_sets = {"carc_name": "*", "representation": "product"}
# This will pull data for all products instead of specific ones

# Define date range
start_date = "2022-11-25"
end_date = "2025-11-24"

# Or use relative dates:
# end_date = datetime.now().strftime("%Y-%m-%d")
# start_date = (datetime.now() - timedelta(days=1095)).strftime("%Y-%m-%d")  # 3 years

# Display entity configuration
if isinstance(entity_sets, dict) and entity_sets.get("carc_name") == "*":
    print(f"✅ Using wildcard: All {entity_sets.get('representation', 'entities')}")
else:
    print(f"✅ Defined {len(entity_sets)} entities")
print(f"✅ Date range: {start_date} to {end_date}")

Step 3: Create Frameworks for Clickstream Core Panel Insights​

Create frameworks for each of the four Clickstream Core Panel insights.

Note: Clickstream frameworks include a platform_name filter that can be set to "*" for all platforms or a specific platform name.

# Framework 1: Website Traffic (Insight ID 379)
framework_traffic = client.explorer.build_framework(
    entities=entity_sets,
    insight=379,  # Website Traffic
    filters={
        "location_resolution": "ww",  # Options: "us", "ww", or specific country codes
        "date_resolution": "day",  # Options: "day", "week", "month", "quarter", "year"
        "platform_name": "*",  # Use "*" for all platforms, or specify a platform name "Desktop" or "Mobile"
        "date_range": {
            "start_date": start_date,
            "end_date": end_date
        }
    },
    aggregate="sum"  # Use "sum" for website traffic
)

print("✅ Framework 1: Website Traffic (379)")
print(f"   Insight ID: {framework_traffic['insight']['insight_id']}")
print(f"   Aggregate: {framework_traffic['aggregate']}")
print(f"   Platform: {framework_traffic['filters']['platform_name']}")
# Framework 2: Average Website Traffic (Insight ID 380)
framework_avg_traffic = client.explorer.build_framework(
    entities=entity_sets,
    insight=380,  # Average Website Traffic
    filters={
        "location_resolution": "ww",
        "date_resolution": "day",
        "platform_name": "*",  # Use "*" for all platforms or specify a platform name "Desktop" or "Mobile"
        "date_range": {
            "start_date": start_date,
            "end_date": end_date
        }
    },
    aggregate="mean"  # Use "mean" for average traffic
)

print("✅ Framework 2: Average Website Traffic (380)")
print(f"   Insight ID: {framework_avg_traffic['insight']['insight_id']}")
print(f"   Aggregate: {framework_avg_traffic['aggregate']}")
print(f"   Platform: {framework_avg_traffic['filters']['platform_name']}")
# Framework 3: Website Traffic Per User (Insight ID 91)
framework_traffic_per_user = client.explorer.build_framework(
    entities=entity_sets,
    insight=91,  # Website Traffic Per User
    filters={
        "location_resolution": "ww",
        "date_resolution": "day",
        "platform_name": "*",  # Use "*" for all platforms or specify a platform name "Desktop" or "Mobile"
        "date_range": {
            "start_date": start_date,
            "end_date": end_date
        }
    },
    aggregate="mean"  # Use "mean" for per-user metrics
)

print("✅ Framework 3: Website Traffic Per User (91)")
print(f"   Insight ID: {framework_traffic_per_user['insight']['insight_id']}")
print(f"   Aggregate: {framework_traffic_per_user['aggregate']}")
print(f"   Platform: {framework_traffic_per_user['filters']['platform_name']}")
# Framework 4: Website Users (Insight ID 381)
framework_users = client.explorer.build_framework(
    entities=entity_sets,
    insight=381,  # Website Users
    filters={
        "location_resolution": "ww",
        "date_resolution": "day",
        "platform_name": "*",  # Use "*" for all platforms or specify a platform name "Desktop" or "Mobile"
        "date_range": {
            "start_date": start_date,
            "end_date": end_date
        }
    },
    aggregate="sum"  # Use "sum" for user count
)

print("✅ Framework 4: Website Users (381)")
print(f"   Insight ID: {framework_users['insight']['insight_id']}")
print(f"   Aggregate: {framework_users['aggregate']}")
print(f"   Platform: {framework_users['filters']['platform_name']}")
# Check prices for all frameworks
print("💰 Checking framework prices...\n")

frameworks_to_check = {
    'Website Traffic (379)': framework_traffic,
    'Average Website Traffic (380)': framework_avg_traffic,
    'Website Traffic Per User (91)': framework_traffic_per_user,
    'Website Users (381)': framework_users
}
prices = {}
total_price = 0
for name, framework in frameworks_to_check.items():
    try:
        price = client.explorer.check_framework_price(framework)
        if price is not None:
            prices[name] = price
            total_price += price
            print(f"✅ {name}: ${price}")
        else:
            print(f"ℹ️ {name}: Price information not available")
            prices[name] = None
    except Exception as e:
        print(f"⚠️ {name}: Could not retrieve price - {str(e)}")
        prices[name] = None

if total_price > 0:
    print(f"\n💰 Total price for all frameworks: ${total_price}")
else:
    print("\nℹ️ Total price calculation skipped (some prices unavailable)")

Step 5: Purchase Frameworks​

Purchase all frameworks at once or individually:

# Option 1: Purchase all frameworks at once
all_frameworks = [
    framework_traffic,
    framework_avg_traffic,
    framework_traffic_per_user,
    framework_users
]

print("🛒 Purchasing all Clickstream Core Panel frameworks...")
order = client.explorer.buy_frameworks(all_frameworks)

print("✅ Frameworks purchased successfully")
print(f"   Framework IDs: {order['frameworks']}")

# Store framework IDs
framework_ids = {
    'traffic': order['frameworks'][0],           # 379
    'avg_traffic': order['frameworks'][1],      # 380
    'traffic_per_user': order['frameworks'][2], # 91
    'users': order['frameworks'][3]             # 381
}

print("\n💡 Framework IDs saved:")
for key, fid in framework_ids.items():
    print(f"   {key}: {fid}")

Option 2: Purchase Frameworks Individually​

If you prefer to purchase frameworks one at a time:

Option 2: Purchase frameworks individually

Uncomment the frameworks you want to purchase

framework_ids = {}

# Purchase Website Traffic (379)
# order_traffic = client.explorer.buy_frameworks(framework_traffic)
# framework_ids['traffic'] = order_traffic['frameworks'][0]
# print(f"✅ Purchased Website Traffic: {framework_ids['traffic']}")

# Purchase Average Website Traffic (380)
# order_avg_traffic = client.explorer.buy_frameworks(framework_avg_traffic)
# framework_ids['avg_traffic'] = order_avg_traffic['frameworks'][0]
# print(f"✅ Purchased Average Website Traffic: {framework_ids['avg_traffic']}")

# Purchase Website Traffic Per User (91)
# order_traffic_per_user = client.explorer.buy_frameworks(framework_traffic_per_user)
# framework_ids['traffic_per_user'] = order_traffic_per_user['frameworks'][0]
# print(f"✅ Purchased Website Traffic Per User: {framework_ids['traffic_per_user']}")

# Purchase Website Users (381)
# order_users = client.explorer.buy_frameworks(framework_users)
# framework_ids['users'] = order_users['frameworks'][0]
# print(f"✅ Purchased Website Users: {framework_ids['users']}")

print("💡 Uncomment the frameworks above to purchase them individually")

Step 6: Check Framework Status​

Check the status of your frameworks (they may need time to process):

# Check status of all frameworks
print("📊 Checking framework status...")
for name, framework_id in framework_ids.items():
    status = client.explorer.get_framework_status(framework_id)
    print(f"   {name} ({framework_id}): {status.get('status', 'unknown')}")

Step 7: Retrieve Data​

Once frameworks are ready, retrieve data for all insights:

# Retrieve data for all frameworks
dataframes = {}

# Website Traffic (379)
df_traffic = client.explorer.get_framework_data(
    framework_id=framework_ids['traffic'],
    data_type="dataframe",
    fetch_all=True
)
dataframes['traffic'] = df_traffic
print(f"✅ Website Traffic: {df_traffic.shape[0]} rows")

# Average Website Traffic (380)
df_avg_traffic = client.explorer.get_framework_data(
    framework_id=framework_ids['avg_traffic'],
    data_type="dataframe",
    fetch_all=True
)
dataframes['avg_traffic'] = df_avg_traffic
print(f"✅ Average Website Traffic: {df_avg_traffic.shape[0]} rows")

# Website Traffic Per User (91)
df_traffic_per_user = client.explorer.get_framework_data(
    framework_id=framework_ids['traffic_per_user'],
    data_type="dataframe",
    fetch_all=True
)
dataframes['traffic_per_user'] = df_traffic_per_user
print(f"✅ Website Traffic Per User: {df_traffic_per_user.shape[0]} rows")

# Website Users (381)
df_users = client.explorer.get_framework_data(
    framework_id=framework_ids['users'],
    data_type="dataframe",
    fetch_all=True
)
dataframes['users'] = df_users
print(f"✅ Website Users: {df_users.shape[0]} rows")

Step 8: View and Export Data​

View sample data and export to CSV:

# Display sample data for each framework
print("📊 Website Traffic (379):")
print(dataframes['traffic'].head())
print(f"\n📊 Average Website Traffic (380):")
print(dataframes['avg_traffic'].head())
print(f"\n📊 Website Traffic Per User (91):")
print(dataframes['traffic_per_user'].head())
print(f"\n📊 Website Users (381):")
print(dataframes['users'].head())
# Export all dataframes to CSV
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')

for name, df in dataframes.items():
    filename = f"clickstream_{name}_{timestamp}.csv"
    df.to_csv(filename, index=False)
    print(f"💾 Exported {name} to {filename}")

Summary​

This guide covered creating frameworks for all four Clickstream Core Panel insights:

  1. ✅ Website Traffic (379) - Use aggregate="sum"
  2. ✅ Average Website Traffic (380) - Use aggregate="mean"
  3. ✅ Website Traffic Per User (91) - Use aggregate="mean"
  4. ✅ Website Users (381) - Use aggregate="sum"

Key Points:​

  • Aggregation: Use "sum" for traffic and users; use "mean" for average metrics
  • Platform Filter: Use platform_name="*" for all platforms, or specify a platform name
  • Date Resolution: Common options are "day", "week", "month", "quarter", "year"
  • Location Resolution: Options include "ww" (worldwide), "us", or specific country codes
  • Processing Time: Frameworks may take time to process, especially for large entity sets

Next Steps:​

  • Combine the dataframes for comprehensive analysis
  • Create visualizations comparing traffic, average traffic, traffic per user, and users
  • Export to S3 or other storage systems for long-term storage