Templat data piutang usaha

/* 
Title: HighRadius Event Log Transformation Script
Description: This script assumes the raw CSV exports from HighRadius have been loaded into four staging tables:
1. STG_INVOICES (Header data)
2. STG_ACTIVITIES (Collections notes, calls, correspondence)
3. STG_DISPUTES (Deduction and dispute lifecycle)
4. STG_PAYMENTS (Cash application details)
*/

/* 1. Invoice Created */
SELECT 
    InvoiceNumber AS InvoiceNumber,
    'Invoice Created' AS ActivityName,
    InvoiceDate AS EventDateTime,
    CustomerName AS CustomerName,
    InvoiceAmount AS InvoiceAmount,
    DueDate AS DueDate,
    'System' AS CollectorName,
    'True' AS IsAutomated,
    NULL AS PaymentTermVariance,
    0 AS DaysSalesOutstanding
FROM STG_INVOICES

UNION ALL

/* 2. Payment Reminder Sent */
SELECT 
    InvoiceNumber,
    'Payment Reminder Sent',
    ActivityDate,
    CustomerName,
    NULL,
    NULL,
    UserName AS CollectorName,
    CASE WHEN UserName = 'System' THEN 'True' ELSE 'False' END,
    NULL,
    NULL
FROM STG_ACTIVITIES
WHERE ActivityType IN ('Dunning', 'Correspondence', 'Email Sent')

UNION ALL

/* 3. Collection Call Logged */
SELECT 
    InvoiceNumber,
    'Collection Call Logged',
    ActivityDate,
    CustomerName,
    NULL,
    NULL,
    UserName AS CollectorName,
    'False' AS IsAutomated,
    NULL,
    NULL
FROM STG_ACTIVITIES
WHERE ActivityType IN ('Call', 'Outbound Call')

UNION ALL

/* 4. Promise to Pay Created */
SELECT 
    InvoiceNumber,
    'Promise to Pay Created',
    PromiseDate,
    CustomerName,
    PromiseAmount,
    NULL,
    UserName AS CollectorName,
    'False' AS IsAutomated,
    NULL,
    NULL
FROM STG_ACTIVITIES
WHERE ActivityType = 'Promise to Pay' AND PromiseDate IS NOT NULL

UNION ALL

/* 5. Dispute Case Opened */
SELECT 
    InvoiceNumber,
    'Dispute Case Opened',
    CreateDate,
    CustomerName,
    DisputeAmount,
    NULL,
    AnalystName AS CollectorName,
    'False' AS IsAutomated,
    NULL,
    NULL
FROM STG_DISPUTES

UNION ALL

/* 6. Dispute Under Review */
SELECT 
    InvoiceNumber,
    'Dispute Under Review',
    StatusDate,
    CustomerName,
    NULL,
    NULL,
    AnalystName AS CollectorName,
    'False' AS IsAutomated,
    NULL,
    NULL
FROM STG_DISPUTES
WHERE Status = 'In Review'

UNION ALL

/* 7. Dispute Resolved */
SELECT 
    InvoiceNumber,
    'Dispute Resolved',
    ResolutionDate,
    CustomerName,
    NULL,
    NULL,
    AnalystName AS CollectorName,
    'False' AS IsAutomated,
    NULL,
    NULL
FROM STG_DISPUTES
WHERE Status IN ('Closed', 'Resolved', 'Denied', 'Approved')

UNION ALL

/* 8. Credit Memo Issued */
SELECT 
    InvoiceNumber,
    'Credit Memo Issued',
    PaymentDate,
    CustomerName,
    PaymentAmount,
    NULL,
    'System' AS CollectorName,
    'True' AS IsAutomated,
    NULL,
    NULL
FROM STG_PAYMENTS
WHERE PaymentType = 'Credit Memo'

UNION ALL

/* 9. Partial Payment Posted */
SELECT 
    P.InvoiceNumber,
    'Partial Payment Posted',
    P.PaymentDate,
    I.CustomerName,
    P.PaymentAmount,
    I.DueDate,
    'System' AS CollectorName,
    'True' AS IsAutomated,
    DATEDIFF(day, I.DueDate, P.PaymentDate) AS PaymentTermVariance,
    DATEDIFF(day, I.InvoiceDate, P.PaymentDate) AS DaysSalesOutstanding
FROM STG_PAYMENTS P
JOIN STG_INVOICES I ON P.InvoiceNumber = I.InvoiceNumber
WHERE P.PaymentType = 'Payment' AND P.PaymentAmount < I.InvoiceAmount

UNION ALL

/* 10. Bank Statement Matched */
SELECT 
    InvoiceNumber,
    'Bank Statement Matched',
    MatchDate,
    CustomerName,
    NULL,
    NULL,
    'CashApp System' AS CollectorName,
    'True' AS IsAutomated,
    NULL,
    NULL
FROM STG_PAYMENTS
WHERE MatchDate IS NOT NULL

UNION ALL

/* 11. Full Payment Received */
SELECT 
    P.InvoiceNumber,
    'Full Payment Received',
    P.PaymentDate,
    I.CustomerName,
    P.PaymentAmount,
    I.DueDate,
    'System' AS CollectorName,
    'True' AS IsAutomated,
    DATEDIFF(day, I.DueDate, P.PaymentDate) AS PaymentTermVariance,
    DATEDIFF(day, I.InvoiceDate, P.PaymentDate) AS DaysSalesOutstanding
FROM STG_PAYMENTS P
JOIN STG_INVOICES I ON P.InvoiceNumber = I.InvoiceNumber
WHERE P.PaymentType = 'Payment' AND P.PaymentAmount >= I.InvoiceAmount

UNION ALL

/* 12. Due Date Passed */
SELECT 
    InvoiceNumber,
    'Due Date Passed',
    DueDate,
    CustomerName,
    InvoiceAmount,
    DueDate,
    'System' AS CollectorName,
    'True' AS IsAutomated,
    NULL,
    DATEDIFF(day, InvoiceDate, DueDate) AS DaysSalesOutstanding
FROM STG_INVOICES
WHERE DueDate < GETDATE() AND (ClearDate IS NULL OR ClearDate > DueDate)

UNION ALL

/* 13. Invoice Cleared */
SELECT 
    InvoiceNumber,
    'Invoice Cleared',
    ClearDate,
    CustomerName,
    InvoiceAmount,
    DueDate,
    'System' AS CollectorName,
    'True' AS IsAutomated,
    DATEDIFF(day, DueDate, ClearDate) AS PaymentTermVariance,
    DATEDIFF(day, InvoiceDate, ClearDate) AS DaysSalesOutstanding
FROM STG_INVOICES
WHERE ClearDate IS NOT NULL

import requests
import pandas as pd
import json
from datetime import datetime, timedelta

class=class="hl-string">"hl-comment"># ==========================================
class=class="hl-string">"hl-comment"># CONFIGURATION
class=class="hl-string">"hl-comment"># ==========================================
BASE_URL = class="hl-string">"https://[Your Instance].highradius.com/api/v1"
TOKEN_URL = class="hl-string">"https://[Your Instance].highradius.com/oauth/token"
CLIENT_ID = class="hl-string">"[Your Client ID]"
CLIENT_SECRET = class="hl-string">"[Your Client Secret]"
SCOPE = class="hl-string">"read_write"

class=class="hl-string">"hl-comment"># Extraction Window
START_DATE = class="hl-string">"2023-01-01"
END_DATE = class="hl-string">"2023-01-31"

class=class="hl-string">"hl-comment"># Global container for event log rows
event_log = []

def get_access_token():
    payload = {
        class="hl-string">'grant_type': class="hl-string">'client_credentials',
        class="hl-string">'client_id': CLIENT_ID,
        class="hl-string">'client_secret': CLIENT_SECRET,
        class="hl-string">'scope': SCOPE
    }
    try:
        response = requests.post(TOKEN_URL, data=payload)
        response.raise_for_status()
        return response.json().get(class="hl-string">'access_token')
    except Exception as e:
        print(fclass="hl-string">"Authentication failed: {e}")
        return None

def fetch_paged_data(endpoint, token, params={}):
    headers = {class="hl-string">"Authorization": fclass="hl-string">"Bearer {token}", class="hl-string">"Content-Type": class="hl-string">"application/json"}
    all_records = []
    page = 1
    while True:
        params[class="hl-string">'page'] = page
        params[class="hl-string">'limit'] = 1000
        try:
            resp = requests.get(fclass="hl-string">"{BASE_URL}/{endpoint}", headers=headers, params=params)
            if resp.status_code != 200:
                break
            data = resp.json()
            records = data.get(class="hl-string">'data', [])
            if not records:
                break
            all_records.extend(records)
            page += 1
        except Exception:
            break
    return all_records

def process_extraction():
    token = get_access_token()
    if not token:
        return

    class=class="hl-string">"hl-comment"># 1. Fetch Invoices (Header Data)
    print(class="hl-string">"Fetching Invoices...")
    invoices = fetch_paged_data(class="hl-string">"invoices", token, {class="hl-string">"filter_date_from": START_DATE, class="hl-string">"filter_date_to": END_DATE})
    
    class=class="hl-string">"hl-comment"># Process Invoice Headers and Derived Events
    invoice_ids = []
    for inv in invoices:
        inv_num = str(inv.get(class="hl-string">'invoice_number', class="hl-string">''))
        cust_name = inv.get(class="hl-string">'customer_name', class="hl-string">'')
        amount = float(inv.get(class="hl-string">'amount', 0.0))
        create_date = inv.get(class="hl-string">'create_date')  class=class="hl-string">"hl-comment"># YYYY-MM-DD
        due_date = inv.get(class="hl-string">'due_date')
        clear_date = inv.get(class="hl-string">'clear_date')
        status = inv.get(class="hl-string">'status', class="hl-string">'Open')
        
        invoice_ids.append(inv_num)
        
        class=class="hl-string">"hl-comment"># Event: Invoice Created
        event_log.append({
            class="hl-string">"InvoiceNumber": inv_num,
            class="hl-string">"ActivityName": class="hl-string">"Invoice Created",
            class="hl-string">"EventDateTime": fclass="hl-string">"{create_date} 00:00:00",
            class="hl-string">"CustomerName": cust_name,
            class="hl-string">"InvoiceAmount": amount,
            class="hl-string">"DueDate": due_date,
            class="hl-string">"IsAutomated": True
        })
        
        class=class="hl-string">"hl-comment"># Event: Invoice Cleared
        if status.lower() in [class="hl-string">'closed', class="hl-string">'cleared'] and clear_date:
             event_log.append({
                class="hl-string">"InvoiceNumber": inv_num,
                class="hl-string">"ActivityName": class="hl-string">"Invoice Cleared",
                class="hl-string">"EventDateTime": fclass="hl-string">"{clear_date} 23:59:59",
                class="hl-string">"CustomerName": cust_name,
                class="hl-string">"InvoiceAmount": amount,
                class="hl-string">"DueDate": due_date,
                class="hl-string">"IsAutomated": True
            })
             
        class=class="hl-string">"hl-comment"># Event: Due Date Passed (Calculated)
        class=class="hl-string">"hl-comment"># Logic: If closed and cleared after due date OR if open and today is after due date
        d_due = datetime.strptime(due_date, class="hl-string">"%Y-%m-%d")
        limit_date = datetime.strptime(clear_date, class="hl-string">"%Y-%m-%d") if clear_date else datetime.now()
        
        if limit_date > d_due:
            overdue_ts = d_due + timedelta(days=1)
            event_log.append({
                class="hl-string">"InvoiceNumber": inv_num,
                class="hl-string">"ActivityName": class="hl-string">"Due Date Passed",
                class="hl-string">"EventDateTime": overdue_ts.strftime(class="hl-string">"%Y-%m-%d %H:%M:%S"),
                class="hl-string">"CustomerName": cust_name,
                class="hl-string">"InvoiceAmount": amount,
                class="hl-string">"DueDate": due_date,
                class="hl-string">"IsAutomated": True
            })

        class=class="hl-string">"hl-comment"># Event: Credit Memo Issued
        if inv.get(class="hl-string">'document_type', class="hl-string">'').lower() == class="hl-string">'credit_memo':
             event_log.append({
                class="hl-string">"InvoiceNumber": inv_num,
                class="hl-string">"ActivityName": class="hl-string">"Credit Memo Issued",
                class="hl-string">"EventDateTime": fclass="hl-string">"{create_date} 12:00:00",
                class="hl-string">"CustomerName": cust_name,
                class="hl-string">"InvoiceAmount": amount,
                class="hl-string">"DueDate": due_date,
                class="hl-string">"IsAutomated": True
            })

    class=class="hl-string">"hl-comment"># 2. Fetch Activities / Notes (Calls, Reminders)
    print(class="hl-string">"Fetching Activities...")
    class=class="hl-string">"hl-comment"># In a real scenario, you might iterate invoice_ids in chunks if the API requires filtering by ID
    class=class="hl-string">"hl-comment"># Here we assume an endpoint allowing date filtering similar to invoices
    activities = fetch_paged_data(class="hl-string">"activities", token, {class="hl-string">"date_from": START_DATE, class="hl-string">"date_to": END_DATE})
    
    for act in activities:
        inv_num = str(act.get(class="hl-string">'invoice_number', class="hl-string">''))
        act_type = act.get(class="hl-string">'activity_type', class="hl-string">'').lower()
        act_date = act.get(class="hl-string">'activity_date') class=class="hl-string">"hl-comment"># Assume valid timestamp
        user = act.get(class="hl-string">'performed_by', class="hl-string">'System')
        
        activity_name = None
        if class="hl-string">'email' in act_type or class="hl-string">'letter' in act_type or class="hl-string">'reminder' in act_type:
            activity_name = class="hl-string">"Payment Reminder Sent"
        elif class="hl-string">'call' in act_type:
            activity_name = class="hl-string">"Collection Call Logged"
            
        if activity_name:
            event_log.append({
                class="hl-string">"InvoiceNumber": inv_num,
                class="hl-string">"ActivityName": activity_name,
                class="hl-string">"EventDateTime": act_date,
                class="hl-string">"CollectorName": user,
                class="hl-string">"IsAutomated": user == class="hl-string">'System'
            })

    class=class="hl-string">"hl-comment"># 3. Fetch Promises to Pay
    print(class="hl-string">"Fetching Promises to Pay...")
    ptps = fetch_paged_data(class="hl-string">"promises", token, {class="hl-string">"created_from": START_DATE, class="hl-string">"created_to": END_DATE})
    for ptp in ptps:
        event_log.append({
            class="hl-string">"InvoiceNumber": str(ptp.get(class="hl-string">'invoice_number', class="hl-string">'')),
            class="hl-string">"ActivityName": class="hl-string">"Promise to Pay Created",
            class="hl-string">"EventDateTime": ptp.get(class="hl-string">'created_at'),
            class="hl-string">"CollectorName": ptp.get(class="hl-string">'created_by'),
            class="hl-string">"IsAutomated": False
        })

    class=class="hl-string">"hl-comment"># 4. Fetch Disputes
    print(class="hl-string">"Fetching Disputes...")
    disputes = fetch_paged_data(class="hl-string">"disputes", token, {class="hl-string">"date_from": START_DATE, class="hl-string">"date_to": END_DATE})
    for disp in disputes:
        inv_num = str(disp.get(class="hl-string">'invoice_number', class="hl-string">''))
        create_ts = disp.get(class="hl-string">'create_date')
        status_history = disp.get(class="hl-string">'status_history', [])
        
        class=class="hl-string">"hl-comment"># Event: Dispute Case Opened
        event_log.append({
            class="hl-string">"InvoiceNumber": inv_num,
            class="hl-string">"ActivityName": class="hl-string">"Dispute Case Opened",
            class="hl-string">"EventDateTime": create_ts,
            class="hl-string">"IsAutomated": False
        })
        
        class=class="hl-string">"hl-comment"># Process Audit Trail for Status Changes
        for history in status_history:
            status = history.get(class="hl-string">'status', class="hl-string">'').lower()
            hist_date = history.get(class="hl-string">'date')
            
            if status == class="hl-string">'in_review' or status == class="hl-string">'investigation':
                event_log.append({
                    class="hl-string">"InvoiceNumber": inv_num,
                    class="hl-string">"ActivityName": class="hl-string">"Dispute Under Review",
                    class="hl-string">"EventDateTime": hist_date,
                    class="hl-string">"IsAutomated": False
                })
            elif status == class="hl-string">'resolved' or status == class="hl-string">'closed':
                 event_log.append({
                    class="hl-string">"InvoiceNumber": inv_num,
                    class="hl-string">"ActivityName": class="hl-string">"Dispute Resolved",
                    class="hl-string">"EventDateTime": hist_date,
                    class="hl-string">"IsAutomated": False
                })

    class=class="hl-string">"hl-comment"># 5. Fetch Payments
    print(class="hl-string">"Fetching Payments...")
    payments = fetch_paged_data(class="hl-string">"payments", token, {class="hl-string">"date_from": START_DATE, class="hl-string">"date_to": END_DATE})
    for pay in payments:
        inv_num = str(pay.get(class="hl-string">'invoice_number', class="hl-string">''))
        pay_date = pay.get(class="hl-string">'payment_date')
        pay_amount = float(pay.get(class="hl-string">'payment_amount', 0.0))
        inv_total = float(pay.get(class="hl-string">'invoice_total_amount', 0.0))
        is_matched = pay.get(class="hl-string">'is_matched', False)
        
        class=class="hl-string">"hl-comment"># Determine Partial vs Full
        class=class="hl-string">"hl-comment"># Note: logic simplified; usually compare accumulated payments
        if pay_amount < inv_total:
            act_name = class="hl-string">"Partial Payment Posted"
        else:
            act_name = class="hl-string">"Full Payment Received"
            
        event_log.append({
            class="hl-string">"InvoiceNumber": inv_num,
            class="hl-string">"ActivityName": act_name,
            class="hl-string">"EventDateTime": pay_date,
            class="hl-string">"InvoiceAmount": inv_total,
            class="hl-string">"IsAutomated": True
        })
        
        class=class="hl-string">"hl-comment"># Event: Bank Statement Matched
        class=class="hl-string">"hl-comment"># Trigger if the system flags this payment as successfully matched to bank line
        if is_matched:
             event_log.append({
                class="hl-string">"InvoiceNumber": inv_num,
                class="hl-string">"ActivityName": class="hl-string">"Bank Statement Matched",
                class="hl-string">"EventDateTime": pay_date, class=class="hl-string">"hl-comment"># Often same time as posting
                class="hl-string">"IsAutomated": True
            })

    class=class="hl-string">"hl-comment"># Export
    df = pd.DataFrame(event_log)
    class=class="hl-string">"hl-comment"># Ensure required columns exist even if empty
    required_cols = [class="hl-string">"InvoiceNumber", class="hl-string">"ActivityName", class="hl-string">"EventDateTime"]
    for col in required_cols:
        if col not in df.columns:
            df[col] = None
            
    class=class="hl-string">"hl-comment"># Fill recommended cols with None if missing
    recommended_cols = [class="hl-string">"CustomerName", class="hl-string">"InvoiceAmount", class="hl-string">"DueDate", class="hl-string">"CollectorName", class="hl-string">"IsAutomated"]
    for col in recommended_cols:
        if col not in df.columns:
            df[col] = None

    df.dropna(subset=[class="hl-string">"InvoiceNumber", class="hl-string">"ActivityName", class="hl-string">"EventDateTime"], inplace=True)
    df.sort_values(by=[class="hl-string">"InvoiceNumber", class="hl-string">"EventDateTime"], inplace=True)
    
    output_file = class="hl-string">"HighRadius_EventLog.csv"
    df.to_csv(output_file, index=False)
    print(fclass="hl-string">"Extraction Complete. {len(df)} rows saved to {output_file}")

if __name__ == class="hl-string">"__main__":
    process_extraction()