您的线索到收款数据模板

import pandas as pd

class=class="hl-string">"hl-comment"># --- 1. Load Data ---
class=class="hl-string">"hl-comment"># Load Pipedrive exports. Replace class="hl-string">'deals.csv' and class="hl-string">'activities.csv' with your actual file paths.
deals_df = pd.read_csv(class="hl-string">'deals.csv')
activities_df = pd.read_csv(class="hl-string">'activities.csv')

class=class="hl-string">"hl-comment"># Placeholder: Load data from external systems. These files should contain a Deal ID column.
class=class="hl-string">"hl-comment"># Replace with actual file loads from your ERP, Accounting, or Fulfillment systems.
sales_orders_df = pd.DataFrame({class="hl-string">'Deal ID': [], class="hl-string">'Order Created Date': []}) class=class="hl-string">"hl-comment"># Example from ERP
fulfillments_df = pd.DataFrame({class="hl-string">'Deal ID': [], class="hl-string">'Fulfillment Date': []})   class=class="hl-string">"hl-comment"># Example from Fulfillment System
invoices_df = pd.DataFrame({class="hl-string">'Deal ID': [], class="hl-string">'Invoice Sent Date': []})         class=class="hl-string">"hl-comment"># Example from Accounting System
payments_df = pd.DataFrame({class="hl-string">'Deal ID': [], class="hl-string">'Payment Received Date': []})     class=class="hl-string">"hl-comment"># Example from Accounting System

class=class="hl-string">"hl-comment"># --- 2. Prepare and Clean Data ---
class=class="hl-string">"hl-comment"># Rename columns for consistency and convert date columns to datetime objects
deals_df.rename(columns={class="hl-string">'ID': class="hl-string">'SalesOpportunityId', class="hl-string">'Value': class="hl-string">'DealValue', class="hl-string">'Owner name': class="hl-string">'DealOwner', class="hl-string">'Status': class="hl-string">'DealStatus', class="hl-string">'Add time': class="hl-string">'EventTime_Created', class="hl-string">'Won time': class="hl-string">'EventTime_Won', class="hl-string">'Lost time': class="hl-string">'EventTime_Lost'}, inplace=True)
activities_df.rename(columns={class="hl-string">'Deal ID': class="hl-string">'SalesOpportunityId', class="hl-string">'Subject': class="hl-string">'ActivityName', class="hl-string">'Done time': class="hl-string">'EventTime'}, inplace=True)

for col in [class="hl-string">'EventTime_Created', class="hl-string">'EventTime_Won', class="hl-string">'EventTime_Lost']: 
    deals_df[col] = pd.to_datetime(deals_df[col], errors=class="hl-string">'coerce')
activities_df[class="hl-string">'EventTime'] = pd.to_datetime(activities_df[class="hl-string">'EventTime'], errors=class="hl-string">'coerce')


class=class="hl-string">"hl-comment"># --- 3. Extract Activities from Pipedrive Data ---

class=class="hl-string">"hl-comment"># Activity: Deal Created
event_created = deals_df[[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime_Created']].copy()
event_created.rename(columns={class="hl-string">'EventTime_Created': class="hl-string">'EventTime'}, inplace=True)
event_created[class="hl-string">'ActivityName'] = class="hl-string">'Deal Created'

class=class="hl-string">"hl-comment"># Activity: Deal Won
event_won = deals_df[deals_df[class="hl-string">'EventTime_Won'].notna()][[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime_Won']].copy()
event_won.rename(columns={class="hl-string">'EventTime_Won': class="hl-string">'EventTime'}, inplace=True)
event_won[class="hl-string">'ActivityName'] = class="hl-string">'Deal Won'

class=class="hl-string">"hl-comment"># Activity: Deal Lost
event_lost = deals_df[deals_df[class="hl-string">'EventTime_Lost'].notna()][[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime_Lost']].copy()
event_lost.rename(columns={class="hl-string">'EventTime_Lost': class="hl-string">'EventTime'}, inplace=True)
event_lost[class="hl-string">'ActivityName'] = class="hl-string">'Deal Lost'

class=class="hl-string">"hl-comment"># Filter for relevant, completed activities linked to a deal
completed_activities = activities_df[activities_df[class="hl-string">'EventTime'].notna() & activities_df[class="hl-string">'SalesOpportunityId'].notna()].copy()
completed_activities[class="hl-string">'SalesOpportunityId'] = completed_activities[class="hl-string">'SalesOpportunityId'].astype(int)

class=class="hl-string">"hl-comment"># Activity: Initial Contact Made (first completed activity for each deal)
event_initial_contact = completed_activities.sort_values(class="hl-string">'EventTime', ascending=True).drop_duplicates(class="hl-string">'SalesOpportunityId')
event_initial_contact = event_initial_contact[[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime']].copy()
event_initial_contact[class="hl-string">'ActivityName'] = class="hl-string">'Initial Contact Made'

class=class="hl-string">"hl-comment"># Infer other activities based on activity subject keywords. Adjust keywords as needed.
event_qualified = completed_activities[completed_activities[class="hl-string">'ActivityName'].str.contains(class="hl-string">'qualify|qualification', case=False, na=False)].copy()
event_qualified[class="hl-string">'ActivityName'] = class="hl-string">'Deal Qualified'

event_proposal_sent = completed_activities[completed_activities[class="hl-string">'ActivityName'].str.contains(class="hl-string">'proposal|quote', case=False, na=False)].copy()
event_proposal_sent[class="hl-string">'ActivityName'] = class="hl-string">'Proposal Sent'

class=class="hl-string">"hl-comment"># Proposal Follow-Up: Any call or email activity after a proposal was sent
if not event_proposal_sent.empty:
    merged_followup = pd.merge(completed_activities, event_proposal_sent[[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime']].rename(columns={class="hl-string">'EventTime':class="hl-string">'ProposalTime'}), on=class="hl-string">'SalesOpportunityId')
    follow_ups = merged_followup[merged_followup[class="hl-string">'EventTime'] > merged_followup[class="hl-string">'ProposalTime']]
    event_follow_up = follow_ups[follow_ups[class="hl-string">'ActivityName'].str.contains(class="hl-string">'call|email|follow-up', case=False, na=False)].copy()
    event_follow_up[class="hl-string">'ActivityName'] = class="hl-string">'Proposal Follow-Up'
else:
    event_follow_up = pd.DataFrame(columns=[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime', class="hl-string">'ActivityName'])

event_negotiation = completed_activities[completed_activities[class="hl-string">'ActivityName'].str.contains(class="hl-string">'negotiation|negotiate', case=False, na=False)].copy()
event_negotiation[class="hl-string">'ActivityName'] = class="hl-string">'Negotiation Started'

class=class="hl-string">"hl-comment"># Placeholder for Deal Reopened: Logic is complex and not reliably available from standard export.
event_reopened = pd.DataFrame(columns=[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime', class="hl-string">'ActivityName'])

class=class="hl-string">"hl-comment"># --- 4. Extract Activities from External Data (Placeholders) ---

class=class="hl-string">"hl-comment"># Rename columns and convert dates for external data
class=class="hl-string">"hl-comment"># This section needs to be adapted to your specific file structures
event_so_created = sales_orders_df.rename(columns={class="hl-string">'Deal ID': class="hl-string">'SalesOpportunityId', class="hl-string">'Order Created Date': class="hl-string">'EventTime'}).assign(ActivityName=class="hl-string">'Sales Order Created')
event_fulfilled = fulfillments_df.rename(columns={class="hl-string">'Deal ID': class="hl-string">'SalesOpportunityId', class="hl-string">'Fulfillment Date': class="hl-string">'EventTime'}).assign(ActivityName=class="hl-string">'Order Fulfilled')
event_invoice_sent = invoices_df.rename(columns={class="hl-string">'Deal ID': class="hl-string">'SalesOpportunityId', class="hl-string">'Invoice Sent Date': class="hl-string">'EventTime'}).assign(ActivityName=class="hl-string">'Invoice Sent')
event_payment_received = payments_df.rename(columns={class="hl-string">'Deal ID': class="hl-string">'SalesOpportunityId', class="hl-string">'Payment Received Date': class="hl-string">'EventTime'}).assign(ActivityName=class="hl-string">'Payment Received')


class=class="hl-string">"hl-comment"># --- 5. Combine and Finalize Event Log ---

event_log_frames = [
    event_created,
    event_won,
    event_lost,
    event_initial_contact,
    event_qualified[[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime', class="hl-string">'ActivityName']],
    event_proposal_sent[[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime', class="hl-string">'ActivityName']],
    event_follow_up[[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime', class="hl-string">'ActivityName']],
    event_negotiation[[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime', class="hl-string">'ActivityName']],
    event_reopened,
    event_so_created,
    event_fulfilled,
    event_invoice_sent,
    event_payment_received
]

event_log = pd.concat(event_log_frames)

class=class="hl-string">"hl-comment"># Merge with deal attributes
final_event_log = pd.merge(event_log, deals_df[[class="hl-string">'SalesOpportunityId', class="hl-string">'DealValue', class="hl-string">'DealOwner', class="hl-string">'Status']], on=class="hl-string">'SalesOpportunityId', how=class="hl-string">'left')

class=class="hl-string">"hl-comment"># Clean up and select final columns
final_event_log.rename(columns={class="hl-string">'Status':class="hl-string">'DealStatus'}, inplace=True)
final_event_log.dropna(subset=[class="hl-string">'EventTime', class="hl-string">'SalesOpportunityId'], inplace=True)

class=class="hl-string">"hl-comment"># Placeholder for DiscountPercentage - Assuming it's a custom field in your deals export
if class="hl-string">'Discount percentage' in deals_df.columns:
    final_event_log = pd.merge(final_event_log, deals_df[[class="hl-string">'SalesOpportunityId', class="hl-string">'Discount percentage']], on=class="hl-string">'SalesOpportunityId', how=class="hl-string">'left')
    final_event_log.rename(columns={class="hl-string">'Discount percentage':class="hl-string">'DiscountPercentage'}, inplace=True)
else:
    final_event_log[class="hl-string">'DiscountPercentage'] = None

class=class="hl-string">"hl-comment"># Reorder columns to match requirements
output_columns = [class="hl-string">'SalesOpportunityId', class="hl-string">'ActivityName', class="hl-string">'EventTime', class="hl-string">'DealValue', class="hl-string">'DealOwner', class="hl-string">'DealStatus', class="hl-string">'DiscountPercentage']
final_event_log = final_event_log[output_columns]

class=class="hl-string">"hl-comment"># --- 6. Save Final Event Log ---
final_event_log.to_csv(class="hl-string">'pipedrive_event_log.csv', index=False, date_format=class="hl-string">'%Y-%m-%d %H:%M:%S')

print(class="hl-string">'Event log created successfully.')

import requests
import pandas as pd
import time
import datetime

class=class="hl-string">"hl-comment"># --- CONFIGURATION ---
class=class="hl-string">"hl-comment"># Replace with your Pipedrive API token and company domain
API_TOKEN = class="hl-string">'YOUR_API_TOKEN'
COMPANY_DOMAIN = class="hl-string">'YOUR_COMPANY_DOMAIN' class=class="hl-string">"hl-comment"># e.g., class="hl-string">'your-company-name'

class=class="hl-string">"hl-comment"># Set the start date for deal extraction (YYYY-MM-DD)
START_DATE = class="hl-string">'2023-01-01'

class=class="hl-string">"hl-comment"># Configure your specific Pipeline and Stage IDs
class=class="hl-string">"hl-comment"># Find these in your Pipedrive URL or via the API /stages endpoint
PIPELINE_ID = 1 class=class="hl-string">"hl-comment"># Replace with your main sales pipeline ID
QUALIFIED_STAGE_ID = 2 class=class="hl-string">"hl-comment"># Replace with your class="hl-string">'Qualified' stage ID
PROPOSAL_STAGE_ID = 3 class=class="hl-string">"hl-comment"># Replace with your class="hl-string">'Proposal Sent' stage ID
NEGOTIATION_STAGE_ID = 4 class=class="hl-string">"hl-comment"># Replace with your class="hl-string">'Negotiation' stage ID

class=class="hl-string">"hl-comment"># --- SCRIPT --- 

BASE_URL = fclass="hl-string">'https://{COMPANY_DOMAIN}.pipedrive.com/api/v1'

def get_paginated_data(endpoint, params={}):
    class="hl-string">"""Fetches all pages of data from a Pipedrive API endpoint."""
    all_data = []
    start = 0
    limit = 500
    while True:
        request_params = {
            class="hl-string">'start': start,
            class="hl-string">'limit': limit,
            class="hl-string">'api_token': API_TOKEN
        }
        request_params.update(params)
        
        try:
            response = requests.get(fclass="hl-string">"{BASE_URL}/{endpoint}", params=request_params)
            response.raise_for_status()
            data = response.json()

            if not data.get(class="hl-string">'data'):
                break

            all_data.extend(data[class="hl-string">'data'])

            if not data.get(class="hl-string">'additional_data', {}).get(class="hl-string">'pagination', {}).get(class="hl-string">'more_items_in_collection'):
                break
            
            start += limit
            time.sleep(0.2) class=class="hl-string">"hl-comment"># Respect API rate limits
        except requests.exceptions.RequestException as e:
            print(fclass="hl-string">"Error fetching {endpoint}: {e}")
            break
            
    return all_data

def get_deal_flow(deal_id):
    class="hl-string">"""Fetches the change history (flow) for a single deal."""
    params = {class="hl-string">'api_token': API_TOKEN}
    try:
        response = requests.get(fclass="hl-string">"{BASE_URL}/deals/{deal_id}/flow", params=params)
        response.raise_for_status()
        data = response.json()
        return data.get(class="hl-string">'data', [])
    except requests.exceptions.RequestException as e:
        print(fclass="hl-string">"Error fetching flow for deal {deal_id}: {e}")
        return []

class=class="hl-string">"hl-comment"># Placeholder functions for external data
def get_sales_order_data(deal_ids):
    print(class="hl-string">"Placeholder: Fetching sales order data from external ERP/CRM...")
    class=class="hl-string">"hl-comment"># Example: Connect to your ERP, query for orders linked to deal_ids
    class=class="hl-string">"hl-comment"># Return a list of dicts: [{class="hl-string">'SalesOpportunityId': id, class="hl-string">'ActivityName': class="hl-string">'Sales Order Created', class="hl-string">'EventTime': timestamp}, ...]
    return []

def get_fulfillment_data(deal_ids):
    print(class="hl-string">"Placeholder: Fetching order fulfillment data...")
    return []

def get_invoice_data(deal_ids):
    print(class="hl-string">"Placeholder: Fetching invoice data from accounting system...")
    return []

def get_payment_data(deal_ids):
    print(class="hl-string">"Placeholder: Fetching payment data from financial system...")
    return []


class=class="hl-string">"hl-comment"># --- MAIN EXTRACTION LOGIC ---
if __name__ == class="hl-string">'__main__':
    print(class="hl-string">"Starting Pipedrive data extraction...")
    events = []

    class=class="hl-string">"hl-comment"># 1. Fetch all deals created after the START_DATE and in the specified pipeline
    print(class="hl-string">"Fetching deals...")
    deals = get_paginated_data(class="hl-string">'deals', {class="hl-string">'start_date': START_DATE, class="hl-string">'pipeline_id': PIPELINE_ID})
    if not deals:
        print(class="hl-string">"No deals found for the specified criteria. Exiting.")
        exit()

    deal_ids = [d[class="hl-string">'id'] for d in deals]
    print(fclass="hl-string">"Found {len(deals)} deals.")

    class=class="hl-string">"hl-comment"># 2. Fetch all activities related to the deals
    print(class="hl-string">"Fetching activities...")
    all_activities = get_paginated_data(class="hl-string">'activities', {class="hl-string">'user_id': 0}) class=class="hl-string">"hl-comment"># Get for all users
    deal_activities = [act for act in all_activities if act.get(class="hl-string">'deal_id') in deal_ids and act[class="hl-string">'done']]

    activities_by_deal = {}
    for act in deal_activities:
        if act[class="hl-string">'deal_id'] not in activities_by_deal:
            activities_by_deal[act[class="hl-string">'deal_id']] = []
        activities_by_deal[act[class="hl-string">'deal_id']].append(act)

    class=class="hl-string">"hl-comment"># 3. Process each deal to create events
    print(class="hl-string">"Processing deals and activities to generate event log...")
    for deal in deals:
        deal_id = deal[class="hl-string">'id']
        deal_owner_name = deal.get(class="hl-string">'owner_name', class="hl-string">'')
        base_attributes = {
            class="hl-string">'SalesOpportunityId': deal_id,
            class="hl-string">'DealValue': deal.get(class="hl-string">'value', 0),
            class="hl-string">'DealOwner': deal_owner_name,
            class="hl-string">'DealStatus': deal.get(class="hl-string">'status', class="hl-string">'open'),
            class="hl-string">'DiscountPercentage': deal.get(class="hl-string">'discount_percentage', 0)
        }

        class=class="hl-string">"hl-comment"># Activity: Deal Created
        events.append({**base_attributes, class="hl-string">'ActivityName': class="hl-string">'Deal Created', class="hl-string">'EventTime': deal[class="hl-string">'add_time']})

        class=class="hl-string">"hl-comment"># Activity: Deal Won / Lost
        if deal[class="hl-string">'status'] == class="hl-string">'won' and deal[class="hl-string">'won_time']:
            events.append({**base_attributes, class="hl-string">'ActivityName': class="hl-string">'Deal Won', class="hl-string">'EventTime': deal[class="hl-string">'won_time']})
        elif deal[class="hl-string">'status'] == class="hl-string">'lost' and deal[class="hl-string">'lost_time']:
            events.append({**base_attributes, class="hl-string">'ActivityName': class="hl-string">'Deal Lost', class="hl-string">'EventTime': deal[class="hl-string">'lost_time']})

        class=class="hl-string">"hl-comment"># Activity: Initial Contact Made & Proposal Follow-Up
        proposal_sent_time = None
        if deal_id in activities_by_deal:
            sorted_activities = sorted(activities_by_deal[deal_id], key=lambda x: x[class="hl-string">'marked_as_done_time'])
            if sorted_activities:
                first_activity = sorted_activities[0]
                events.append({**base_attributes, class="hl-string">'ActivityName': class="hl-string">'Initial Contact Made', class="hl-string">'EventTime': first_activity[class="hl-string">'marked_as_done_time']})
            
            class=class="hl-string">"hl-comment"># Find proposal follow-ups (requires proposal time)
            class=class="hl-string">"hl-comment"># This part is revisited after stage changes are processed

        class=class="hl-string">"hl-comment"># Activities from Stage Changes: Qualified, Proposal Sent, Negotiation, Reopened
        time.sleep(0.2) class=class="hl-string">"hl-comment"># Rate limit for flow endpoint
        flow = get_deal_flow(deal_id)
        if not flow:
            continue

        lost_then_reopened = False
        was_lost = False

        for change in sorted(flow, key=lambda x: x[class="hl-string">'add_time']):
            class=class="hl-string">"hl-comment"># Deal Reopened
            if change.get(class="hl-string">'field_key') == class="hl-string">'status' and change.get(class="hl-string">'old_value') == class="hl-string">'lost' and change.get(class="hl-string">'new_value') == class="hl-string">'open':
                events.append({**base_attributes, class="hl-string">'ActivityName': class="hl-string">'Deal Reopened', class="hl-string">'EventTime': change[class="hl-string">'add_time']})
            
            if change.get(class="hl-string">'field_key') != class="hl-string">'stage_id':
                continue

            stage_change_time = change[class="hl-string">'add_time']

            class=class="hl-string">"hl-comment"># Deal Qualified
            if change[class="hl-string">'new_value'] == QUALIFIED_STAGE_ID:
                events.append({**base_attributes, class="hl-string">'ActivityName': class="hl-string">'Deal Qualified', class="hl-string">'EventTime': stage_change_time})

            class=class="hl-string">"hl-comment"># Proposal Sent
            if change[class="hl-string">'new_value'] == PROPOSAL_STAGE_ID:
                events.append({**base_attributes, class="hl-string">'ActivityName': class="hl-string">'Proposal Sent', class="hl-string">'EventTime': stage_change_time})
                proposal_sent_time = stage_change_time

            class=class="hl-string">"hl-comment"># Negotiation Started
            if change[class="hl-string">'new_value'] == NEGOTIATION_STAGE_ID:
                events.append({**base_attributes, class="hl-string">'ActivityName': class="hl-string">'Negotiation Started', class="hl-string">'EventTime': stage_change_time})

        class=class="hl-string">"hl-comment"># Re-check for Proposal Follow-Up now that we have proposal_sent_time
        if proposal_sent_time and deal_id in activities_by_deal:
            for act in activities_by_deal[deal_id]:
                if act[class="hl-string">'marked_as_done_time'] > proposal_sent_time:
                    events.append({**base_attributes, class="hl-string">'ActivityName': class="hl-string">'Proposal Follow-Up', class="hl-string">'EventTime': act[class="hl-string">'marked_as_done_time']})
                    break class=class="hl-string">"hl-comment"># Only capture the first follow-up

    class=class="hl-string">"hl-comment"># 4. Add events from external systems (placeholders)
    print(class="hl-string">"Adding data from external system placeholders...")
    events.extend(get_sales_order_data(deal_ids))
    events.extend(get_fulfillment_data(deal_ids))
    events.extend(get_invoice_data(deal_ids))
    events.extend(get_payment_data(deal_ids))

    class=class="hl-string">"hl-comment"># 5. Create and export the final event log
    if not events:
        print(class="hl-string">"No events were generated. The output file will be empty.")
    else:
        print(class="hl-string">"Creating CSV file...")
        df = pd.DataFrame(events)
        class=class="hl-string">"hl-comment"># Ensure required columns are present
        df = df[[class="hl-string">'SalesOpportunityId', class="hl-string">'ActivityName', class="hl-string">'EventTime', class="hl-string">'DealValue', class="hl-string">'DealOwner', class="hl-string">'DealStatus', class="hl-string">'DiscountPercentage']]
        df.sort_values(by=[class="hl-string">'SalesOpportunityId', class="hl-string">'EventTime'], inplace=True)
        
        output_filename = class="hl-string">'pipedrive_event_log.csv'
        df.to_csv(output_filename, index=False)
        print(fclass="hl-string">"Extraction complete. Event log saved to {output_filename}")

import os
from datetime import datetime, timezone
import pandas as pd
from sqlalchemy import create_engine, text

CONNECTION_STRING = os.getenv(class="hl-string">"DESTINATION_CONNECTION_STRING", class="hl-string">"[Connection string]")
START_DATE = os.getenv(class="hl-string">"START_DATE", class="hl-string">"[Start date]")
END_DATE = os.getenv(class="hl-string">"END_DATE", class="hl-string">"[End date]")

DEALS_TABLE = class="hl-string">"[Your deals table]"
ACTIVITIES_TABLE = class="hl-string">"[Your activities table]"
DEAL_STAGE_HISTORY_TABLE = class="hl-string">"[Your deal stage history table]"
DEAL_STATUS_HISTORY_TABLE = class="hl-string">"[Your deal status history table]"
SALES_ORDERS_TABLE = class="hl-string">"[Your sales orders table]"
FULFILLMENT_TABLE = class="hl-string">"[Your fulfillment table]"
INVOICES_TABLE = class="hl-string">"[Your invoices table]"
PAYMENTS_TABLE = class="hl-string">"[Your payments table]"

engine = create_engine(CONNECTION_STRING)

sql = fclass="hl-string">"""
WITH
base_deals AS (
    SELECT
        CAST(d.id AS VARCHAR(255)) AS SalesOpportunityId,
        d.id AS deal_source_id,
        d.add_time AS deal_created_time,
        d.value AS DealValue,
        d.owner_name AS DealOwner,
        d.status AS DealStatus,
        d.discount_percentage AS DiscountPercentage
    FROM {DEALS_TABLE} d
    WHERE d.add_time >= :start_date
      AND d.add_time < :end_date
),
completed_activities AS (
    SELECT
        CAST(a.deal_id AS VARCHAR(255)) AS SalesOpportunityId,
        a.id AS activity_source_id,
        a.done AS activity_done,
        a.activity_type,
        a.subject,
        a.done_time AS activity_time,
        a.add_time AS activity_created_time
    FROM {ACTIVITIES_TABLE} a
    WHERE a.deal_id IS NOT NULL
      AND COALESCE(a.done_time, a.add_time) >= :start_date
      AND COALESCE(a.done_time, a.add_time) < :end_date
      AND a.done = TRUE
),
stage_history AS (
    SELECT
        CAST(h.deal_id AS VARCHAR(255)) AS SalesOpportunityId,
        h.stage_name,
        h.changed_at,
        LAG(h.stage_name) OVER (PARTITION BY h.deal_id ORDER BY h.changed_at, h.id) AS previous_stage
    FROM {DEAL_STAGE_HISTORY_TABLE} h
    WHERE h.changed_at >= :start_date
      AND h.changed_at < :end_date
),
status_history AS (
    SELECT
        CAST(h.deal_id AS VARCHAR(255)) AS SalesOpportunityId,
        LOWER(h.status) AS status,
        LOWER(LAG(h.status) OVER (PARTITION BY h.deal_id ORDER BY h.changed_at, h.id)) AS previous_status,
        h.changed_at
    FROM {DEAL_STATUS_HISTORY_TABLE} h
    WHERE h.changed_at >= :start_date
      AND h.changed_at < :end_date
),
external_sales_orders AS (
    SELECT
        CAST(o.deal_id AS VARCHAR(255)) AS SalesOpportunityId,
        o.created_at AS event_time,
        o.order_value AS DealValue,
        o.owner_name AS DealOwner,
        o.status AS DealStatus,
        o.discount_percentage AS DiscountPercentage
    FROM {SALES_ORDERS_TABLE} o
    WHERE o.deal_id IS NOT NULL
      AND o.created_at >= :start_date
      AND o.created_at < :end_date
),
external_fulfillment AS (
    SELECT
        CAST(f.deal_id AS VARCHAR(255)) AS SalesOpportunityId,
        f.fulfilled_at AS event_time,
        f.order_value AS DealValue,
        f.owner_name AS DealOwner,
        f.status AS DealStatus,
        f.discount_percentage AS DiscountPercentage
    FROM {FULFILLMENT_TABLE} f
    WHERE f.deal_id IS NOT NULL
      AND f.fulfilled_at >= :start_date
      AND f.fulfilled_at < :end_date
),
external_invoices AS (
    SELECT
        CAST(i.deal_id AS VARCHAR(255)) AS SalesOpportunityId,
        i.sent_at AS event_time,
        i.invoice_value AS DealValue,
        i.owner_name AS DealOwner,
        i.status AS DealStatus,
        i.discount_percentage AS DiscountPercentage
    FROM {INVOICES_TABLE} i
    WHERE i.deal_id IS NOT NULL
      AND i.sent_at >= :start_date
      AND i.sent_at < :end_date
),
external_payments AS (
    SELECT
        CAST(p.deal_id AS VARCHAR(255)) AS SalesOpportunityId,
        p.received_at AS event_time,
        p.payment_value AS DealValue,
        p.owner_name AS DealOwner,
        p.status AS DealStatus,
        p.discount_percentage AS DiscountPercentage
    FROM {PAYMENTS_TABLE} p
    WHERE p.deal_id IS NOT NULL
      AND p.received_at >= :start_date
      AND p.received_at < :end_date
),
events AS (
    SELECT b.SalesOpportunityId, 'Deal Created' AS ActivityName, b.deal_created_time AS EventTime, b.DealValue, b.DealOwner, b.DealStatus, b.DiscountPercentage
    FROM base_deals b
    WHERE b.deal_created_time IS NOT NULL
    UNION ALL
    SELECT a.SalesOpportunityId, 'Initial Contact Made', COALESCE(a.activity_time, a.activity_created_time), b.DealValue, b.DealOwner, b.DealStatus, b.DiscountPercentage
    FROM completed_activities a
    JOIN base_deals b ON b.SalesOpportunityId = a.SalesOpportunityId
    WHERE LOWER(COALESCE(a.activity_type, '')) IN ('call', 'email', 'meeting')
       OR LOWER(COALESCE(a.subject, '')) LIKE '%initial contact%'
    UNION ALL
    SELECT h.SalesOpportunityId, 'Deal Qualified', h.changed_at, b.DealValue, b.DealOwner, b.DealStatus, b.DiscountPercentage
    FROM stage_history h
    JOIN base_deals b ON b.SalesOpportunityId = h.SalesOpportunityId
    WHERE LOWER(COALESCE(h.stage_name, '')) IN ('contact made', 'needs defined', 'qualified')
      AND LOWER(COALESCE(h.previous_stage, '')) IN ('lead in', 'lead', 'new')
    UNION ALL
    SELECT h.SalesOpportunityId, 'Proposal Sent', h.changed_at, b.DealValue, b.DealOwner, b.DealStatus, b.DiscountPercentage
    FROM stage_history h
    JOIN base_deals b ON b.SalesOpportunityId = h.SalesOpportunityId
    WHERE LOWER(COALESCE(h.stage_name, '')) IN ('proposal made', 'quote sent', 'proposal sent')
    UNION ALL
    SELECT a.SalesOpportunityId, 'Proposal Follow-Up', COALESCE(a.activity_time, a.activity_created_time), b.DealValue, b.DealOwner, b.DealStatus, b.DiscountPercentage
    FROM completed_activities a
    JOIN base_deals b ON b.SalesOpportunityId = a.SalesOpportunityId
    WHERE LOWER(COALESCE(a.activity_type, '')) IN ('call', 'email', 'meeting')
      AND COALESCE(a.activity_time, a.activity_created_time) > COALESCE((SELECT MIN(h.changed_at) FROM stage_history h WHERE h.SalesOpportunityId = a.SalesOpportunityId AND LOWER(COALESCE(h.stage_name, '')) IN ('proposal made', 'quote sent', 'proposal sent')), TIMESTAMP '1900-01-01')
    UNION ALL
    SELECT h.SalesOpportunityId, 'Negotiation Started', h.changed_at, b.DealValue, b.DealOwner, b.DealStatus, b.DiscountPercentage
    FROM stage_history h
    JOIN base_deals b ON b.SalesOpportunityId = h.SalesOpportunityId
    WHERE LOWER(COALESCE(h.stage_name, '')) IN ('negotiation', 'negotiating')
      AND EXISTS (SELECT 1 FROM stage_history p WHERE p.SalesOpportunityId = h.SalesOpportunityId AND LOWER(COALESCE(p.stage_name, '')) IN ('proposal made', 'quote sent', 'proposal sent') AND p.changed_at <= h.changed_at)
    UNION ALL
    SELECT h.SalesOpportunityId, 'Deal Won', h.changed_at, b.DealValue, b.DealOwner, b.DealStatus, b.DiscountPercentage
    FROM status_history h
    JOIN base_deals b ON b.SalesOpportunityId = h.SalesOpportunityId
    WHERE h.status = 'won' AND COALESCE(h.previous_status, '') <> 'won'
    UNION ALL
    SELECT o.SalesOpportunityId, 'Sales Order Created', o.event_time, COALESCE(o.DealValue, b.DealValue), COALESCE(o.DealOwner, b.DealOwner), COALESCE(o.DealStatus, b.DealStatus), COALESCE(o.DiscountPercentage, b.DiscountPercentage)
    FROM external_sales_orders o
    LEFT JOIN base_deals b ON b.SalesOpportunityId = o.SalesOpportunityId
    UNION ALL
    SELECT f.SalesOpportunityId, 'Order Fulfilled', f.event_time, COALESCE(f.DealValue, b.DealValue), COALESCE(f.DealOwner, b.DealOwner), COALESCE(f.DealStatus, b.DealStatus), COALESCE(f.DiscountPercentage, b.DiscountPercentage)
    FROM external_fulfillment f
    LEFT JOIN base_deals b ON b.SalesOpportunityId = f.SalesOpportunityId
    UNION ALL
    SELECT i.SalesOpportunityId, 'Invoice Sent', i.event_time, COALESCE(i.DealValue, b.DealValue), COALESCE(i.DealOwner, b.DealOwner), COALESCE(i.DealStatus, b.DealStatus), COALESCE(i.DiscountPercentage, b.DiscountPercentage)
    FROM external_invoices i
    LEFT JOIN base_deals b ON b.SalesOpportunityId = i.SalesOpportunityId
    UNION ALL
    SELECT p.SalesOpportunityId, 'Payment Received', p.event_time, COALESCE(p.DealValue, b.DealValue), COALESCE(p.DealOwner, b.DealOwner), COALESCE(p.DealStatus, b.DealStatus), COALESCE(p.DiscountPercentage, b.DiscountPercentage)
    FROM external_payments p
    LEFT JOIN base_deals b ON b.SalesOpportunityId = p.SalesOpportunityId
    UNION ALL
    SELECT h.SalesOpportunityId, 'Deal Lost', h.changed_at, b.DealValue, b.DealOwner, b.DealStatus, b.DiscountPercentage
    FROM status_history h
    JOIN base_deals b ON b.SalesOpportunityId = h.SalesOpportunityId
    WHERE h.status = 'lost' AND COALESCE(h.previous_status, '') <> 'lost'
    UNION ALL
    SELECT h.SalesOpportunityId, 'Deal Reopened', h.changed_at, b.DealValue, b.DealOwner, b.DealStatus, b.DiscountPercentage
    FROM status_history h
    JOIN base_deals b ON b.SalesOpportunityId = h.SalesOpportunityId
    WHERE h.status = 'open' AND h.previous_status = 'lost'
)
SELECT SalesOpportunityId, ActivityName, EventTime, DealValue, DealOwner, DealStatus, DiscountPercentage
FROM events
WHERE SalesOpportunityId IS NOT NULL
  AND EventTime IS NOT NULL
ORDER BY SalesOpportunityId, EventTime, ActivityName
"""

with engine.connect() as connection:
    event_log = pd.read_sql(text(sql), connection, params={class="hl-string">"start_date": START_DATE, class="hl-string">"end_date": END_DATE})

event_log[class="hl-string">"EventTime"] = pd.to_datetime(event_log[class="hl-string">"EventTime"], utc=True, errors=class="hl-string">"coerce")
event_log = event_log.dropna(subset=[class="hl-string">"SalesOpportunityId", class="hl-string">"ActivityName", class="hl-string">"EventTime"])
event_log = event_log.sort_values([class="hl-string">"SalesOpportunityId", class="hl-string">"EventTime", class="hl-string">"ActivityName"])
event_log.to_csv(class="hl-string">"processmind_lead_to_cash_event_log.csv", index=False)
print(event_log.to_string(index=False))
print(fclass="hl-string">"Exported {len(event_log)} event rows to processmind_lead_to_cash_event_log.csv")