Votre modèle de données de Gestion des Dépenses

-- This is a logical representation of the transformation required to convert the
-- SAP Concur Standard Accounting Extract files into a process mining event log.
-- The source files (e.g., ReportHeader.csv, WorkflowHistory.csv) are the output of the extract.

-- Activity: Expense Report Created
SELECT
    ReportID AS ExpenseReportId,
    'Expense Report Created' AS ActivityName,
    ReportCreateDate AS EventTimestamp,
    EmployeeLoginID AS ProcessorUserId,
    ReportTotal AS ReportTotalAmount,
    EmployeeOrgUnit AS EmployeeDepartment,
    HasException AS PolicyViolationFlag,
    NULL AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM ReportHeader.csv

UNION ALL

-- Activity: Expense Report Submitted
SELECT
    ReportID AS ExpenseReportId,
    'Expense Report Submitted' AS ActivityName,
    ReportSubmitDate AS EventTimestamp,
    EmployeeLoginID AS ProcessorUserId,
    ReportTotal AS ReportTotalAmount,
    EmployeeOrgUnit AS EmployeeDepartment,
    HasException AS PolicyViolationFlag,
    NULL AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM ReportHeader.csv
WHERE ReportSubmitDate IS NOT NULL

UNION ALL

-- Activity: Manager Approved
SELECT
    ReportID AS ExpenseReportId,
    'Manager Approved' AS ActivityName,
    WorkflowStepDate AS EventTimestamp,
    w.ApproverLoginID AS ProcessorUserId,
    h.ReportTotal AS ReportTotalAmount,
    h.EmployeeOrgUnit AS EmployeeDepartment,
    h.HasException AS PolicyViolationFlag,
    w.ApproverLoginID AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM WorkflowHistory.csv w
JOIN ReportHeader.csv h ON w.ReportID = h.ReportID
WHERE w.WorkflowStepAction = 'Approve' AND w.ApproverRole = 'Manager'

UNION ALL

-- Activity: Report Sent Back for Revision
SELECT
    ReportID AS ExpenseReportId,
    'Report Sent Back for Revision' AS ActivityName,
    WorkflowStepDate AS EventTimestamp,
    w.ApproverLoginID AS ProcessorUserId,
    h.ReportTotal AS ReportTotalAmount,
    h.EmployeeOrgUnit AS EmployeeDepartment,
    h.HasException AS PolicyViolationFlag,
    w.ApproverLoginID AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM WorkflowHistory.csv w
JOIN ReportHeader.csv h ON w.ReportID = h.ReportID
WHERE w.WorkflowStepAction = 'Send Back'

UNION ALL

-- Activity: Manager Rejected
SELECT
    ReportID AS ExpenseReportId,
    'Manager Rejected' AS ActivityName,
    WorkflowStepDate AS EventTimestamp,
    w.ApproverLoginID AS ProcessorUserId,
    h.ReportTotal AS ReportTotalAmount,
    h.EmployeeOrgUnit AS EmployeeDepartment,
    h.HasException AS PolicyViolationFlag,
    w.ApproverLoginID AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM WorkflowHistory.csv w
JOIN ReportHeader.csv h ON w.ReportID = h.ReportID
WHERE w.WorkflowStepAction = 'Reject' AND w.ApproverRole = 'Manager'

UNION ALL

-- Activity: Finance Approved
SELECT
    ReportID AS ExpenseReportId,
    'Finance Approved' AS ActivityName,
    WorkflowStepDate AS EventTimestamp,
    w.ApproverLoginID AS ProcessorUserId,
    h.ReportTotal AS ReportTotalAmount,
    h.EmployeeOrgUnit AS EmployeeDepartment,
    h.HasException AS PolicyViolationFlag,
    w.ApproverLoginID AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM WorkflowHistory.csv w
JOIN ReportHeader.csv h ON w.ReportID = h.ReportID
WHERE w.WorkflowStepAction = 'Approve' AND w.ApproverRole = 'Processor'

UNION ALL

-- Activity: Finance Rejected
SELECT
    ReportID AS ExpenseReportId,
    'Finance Rejected' AS ActivityName,
    WorkflowStepDate AS EventTimestamp,
    w.ApproverLoginID AS ProcessorUserId,
    h.ReportTotal AS ReportTotalAmount,
    h.EmployeeOrgUnit AS EmployeeDepartment,
    h.HasException AS PolicyViolationFlag,
    w.ApproverLoginID AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM WorkflowHistory.csv w
JOIN ReportHeader.csv h ON w.ReportID = h.ReportID
WHERE w.WorkflowStepAction = 'Reject' AND w.ApproverRole = 'Processor'

UNION ALL

-- Activity: Reimbursement Scheduled
SELECT
    ReportID AS ExpenseReportId,
    'Reimbursement Scheduled' AS ActivityName,
    PaymentProcessingDate AS EventTimestamp,
    'SYSTEM' AS ProcessorUserId,
    ReportTotal AS ReportTotalAmount,
    EmployeeOrgUnit AS EmployeeDepartment,
    HasException AS PolicyViolationFlag,
    NULL AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM ReportHeader.csv
WHERE PaymentProcessingDate IS NOT NULL

UNION ALL

-- Activity: Reimbursement Executed
SELECT
    ReportID AS ExpenseReportId,
    'Reimbursement Executed' AS ActivityName,
    PaidDate AS EventTimestamp,
    'SYSTEM' AS ProcessorUserId,
    ReportTotal AS ReportTotalAmount,
    EmployeeOrgUnit AS EmployeeDepartment,
    HasException AS PolicyViolationFlag,
    NULL AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM ReportHeader.csv
WHERE PaidDate IS NOT NULL

UNION ALL

-- Activity: Accounting Posted
SELECT
    ReportID AS ExpenseReportId,
    'Accounting Posted' AS ActivityName,
    ExtractDate AS EventTimestamp,
    'SYSTEM' AS ProcessorUserId,
    ReportTotal AS ReportTotalAmount,
    EmployeeOrgUnit AS EmployeeDepartment,
    HasException AS PolicyViolationFlag,
    NULL AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM ReportHeader.csv
WHERE ExtractDate IS NOT NULL

-- The following activities are more complex and may require specific configuration or inference.
-- The logic provided is a common approach.

UNION ALL

-- Activity: Receipts Image Attached (inferred from first submission)
SELECT
    ReportID AS ExpenseReportId,
    'Receipts Image Attached' AS ActivityName,
    ReportSubmitDate AS EventTimestamp, -- This is an approximation; precise time is not in standard extracts
    EmployeeLoginID AS ProcessorUserId,
    ReportTotal AS ReportTotalAmount,
    EmployeeOrgUnit AS EmployeeDepartment,
    HasException AS PolicyViolationFlag,
    NULL AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM ReportHeader.csv
WHERE HasImage = 'Y' AND ReportSubmitDate IS NOT NULL

UNION ALL

-- Activity: Policy Violation Flagged (inferred from exception flag at time of submission)
SELECT
    ReportID AS ExpenseReportId,
    'Policy Violation Flagged' AS ActivityName,
    ReportSubmitDate AS EventTimestamp,
    'SYSTEM' AS ProcessorUserId,
    ReportTotal AS ReportTotalAmount,
    EmployeeOrgUnit AS EmployeeDepartment,
    HasException AS PolicyViolationFlag,
    NULL AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM ReportHeader.csv
WHERE HasException = 'Y' AND ReportSubmitDate IS NOT NULL

UNION ALL

-- Activity: Audit Review Required (inferred from workflow step)
SELECT
    ReportID AS ExpenseReportId,
    'Audit Review Required' AS ActivityName,
    WorkflowStepDate AS EventTimestamp,
    w.ApproverLoginID AS ProcessorUserId,
    h.ReportTotal AS ReportTotalAmount,
    h.EmployeeOrgUnit AS EmployeeDepartment,
    h.HasException AS PolicyViolationFlag,
    w.ApproverLoginID AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM WorkflowHistory.csv w
JOIN ReportHeader.csv h ON w.ReportID = h.ReportID
WHERE w.WorkflowStepName LIKE '%Audit%'

UNION ALL

-- Activity: Finance Review Started (inferred from entering Processor queue)
SELECT
    ReportID AS ExpenseReportId,
    'Finance Review Started' AS ActivityName,
    WorkflowStepDate AS EventTimestamp,
    'SYSTEM' AS ProcessorUserId,
    h.ReportTotal AS ReportTotalAmount,
    h.EmployeeOrgUnit AS EmployeeDepartment,
    h.HasException AS PolicyViolationFlag,
    NULL AS ApproverId,
    NULL AS ExpenseCategory,
    'SAP Concur' as SourceSystem,
    GETDATE() as LastDataUpdate
FROM WorkflowHistory.csv w
JOIN ReportHeader.csv h ON w.ReportID = h.ReportID
WHERE w.WorkflowStepName LIKE '%Processor Review%';

import requests
import pandas as pd
import json
from datetime import datetime

class=class="hl-string">"hl-comment"># --- Configuration Section --- #
class=class="hl-string">"hl-comment"># Replace with your SAP Concur API credentials and settings
CLIENT_ID = class="hl-string">'YOUR_CLIENT_ID'
CLIENT_SECRET = class="hl-string">'YOUR_CLIENT_SECRET'
BASE_URI = class="hl-string">'https://us.api.concursolutions.com' class=class="hl-string">"hl-comment"># Or your specific regional URI

class=class="hl-string">"hl-comment"># Define the date range for the extraction
START_DATE = class="hl-string">'2023-01-01T00:00:00.000Z'
END_DATE = class="hl-string">'2023-06-30T23:59:59.000Z'

class=class="hl-string">"hl-comment"># --- End Configuration Section --- #

def get_access_token():
    class="hl-string">"""Authenticates with Concur API to get an OAuth 2.0 access token."""
    token_url = fclass="hl-string">'{BASE_URI}/oauth2/v0/token'
    headers = {class="hl-string">'Content-Type': class="hl-string">'application/x-www-form-urlencoded'}
    data = {
        class="hl-string">'grant_type': class="hl-string">'client_credentials',
        class="hl-string">'client_id': CLIENT_ID,
        class="hl-string">'client_secret': CLIENT_SECRET
    }
    try:
        response = requests.post(token_url, headers=headers, data=data)
        response.raise_for_status()
        token_data = response.json()
        return token_data.get(class="hl-string">'access_token')
    except requests.exceptions.RequestException as e:
        print(fclass="hl-string">'Error getting access token: {e}')
        return None

def get_reports(token, next_page_url=None):
    class="hl-string">"""Fetches a paginated list of expense reports."""
    if next_page_url:
        url = next_page_url
    else:
        url = fclass="hl-string">'{BASE_URI}/expensereports/v4/reports?lastModifiedDateAfter={START_DATE}&lastModifiedDateBefore={END_DATE}'
    
    headers = {class="hl-string">'Authorization': fclass="hl-string">'Bearer {token}', class="hl-string">'Accept': class="hl-string">'application/json'}
    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        return response.json()
    except requests.exceptions.RequestException as e:
        print(fclass="hl-string">'Error fetching reports list: {e}')
        return None

def get_report_details(token, report_id):
    class="hl-string">"""Fetches the full details for a single expense report."""
    url = fclass="hl-string">'{BASE_URI}/expensereports/v4/reports/{report_id}'
    headers = {class="hl-string">'Authorization': fclass="hl-string">'Bearer {token}', class="hl-string">'Accept': class="hl-string">'application/json'}
    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        return response.json()
    except requests.exceptions.RequestException as e:
        print(fclass="hl-string">'Error fetching details for report {report_id}: {e}')
        return None

def main():
    class="hl-string">"""Main function to orchestrate the extraction process."""
    access_token = get_access_token()
    if not access_token:
        print(class="hl-string">'Could not obtain access token. Exiting.')
        return

    print(class="hl-string">'Successfully obtained access token. Starting report extraction...')
    event_log = []
    last_data_update = datetime.utcnow().isoformat() + class="hl-string">'Z'

    next_page = None
    page_count = 1

    while True:
        print(fclass="hl-string">'Fetching page {page_count} of reports...')
        reports_data = get_reports(access_token, next_page)
        if not reports_data or not reports_data.get(class="hl-string">'value'):
            print(class="hl-string">'No more reports found or API error.')
            break

        for report_summary in reports_data[class="hl-string">'value']:
            report_id = report_summary.get(class="hl-string">'id')
            if not report_id:
                continue

            details = get_report_details(access_token, report_id)
            if not details:
                continue

            class=class="hl-string">"hl-comment"># --- Base Attributes for all events related to this report --- #
            base_event = {
                class="hl-string">'ExpenseReportId': report_id,
                class="hl-string">'ReportTotalAmount': details.get(class="hl-string">'approvedAmount', {}).get(class="hl-string">'value'),
                class="hl-string">'EmployeeDepartment': details.get(class="hl-string">'orgUnit', {}).get(class="hl-string">'name'),
                class="hl-string">'PolicyViolationFlag': details.get(class="hl-string">'hasExceptions'),
                class="hl-string">'SourceSystem': class="hl-string">'SAP Concur',
                class="hl-string">'LastDataUpdate': last_data_update
            }

            class=class="hl-string">"hl-comment"># --- Activity: Expense Report Created --- #
            event_log.append({
                **base_event,
                class="hl-string">'ActivityName': class="hl-string">'Expense Report Created',
                class="hl-string">'EventTimestamp': details.get(class="hl-string">'createDate'),
                class="hl-string">'ProcessorUserId': details.get(class="hl-string">'owner', {}).get(class="hl-string">'id'),
                class="hl-string">'ApproverId': None,
                class="hl-string">'ExpenseCategory': None
            })

            class=class="hl-string">"hl-comment"># --- Activities from Audit Trail --- #
            for audit_entry in details.get(class="hl-string">'auditTrail', []):
                action = audit_entry.get(class="hl-string">'action')
                activity_name = None
                processor = audit_entry.get(class="hl-string">'processor', {}).get(class="hl-string">'id')
                approver = audit_entry.get(class="hl-string">'approver', {}).get(class="hl-string">'id')
                
                if action == class="hl-string">'Submit':
                    activity_name = class="hl-string">'Expense Report Submitted'
                elif action == class="hl-string">'Approve':
                    class=class="hl-string">"hl-comment"># This logic assumes any approval is class="hl-string">'Manager Approved'. 
                    class=class="hl-string">"hl-comment"># A more complex system might check processor roles.
                    activity_name = class="hl-string">'Manager Approved'
                elif action == class="hl-string">'Reject':
                    class=class="hl-string">"hl-comment"># Differentiating manager vs finance rejection may require checking processor role.
                    activity_name = class="hl-string">'Manager Rejected'
                elif action == class="hl-string">'Recall':
                    activity_name = class="hl-string">'Report Sent Back for Revision'
                
                if activity_name:
                    event_log.append({
                        **base_event,
                        class="hl-string">'ActivityName': activity_name,
                        class="hl-string">'EventTimestamp': audit_entry.get(class="hl-string">'entryDate'),
                        class="hl-string">'ProcessorUserId': processor,
                        class="hl-string">'ApproverId': approver,
                        class="hl-string">'ExpenseCategory': None
                    })

            class=class="hl-string">"hl-comment"># --- Activities from Statuses and Dates --- #
            if details.get(class="hl-string">'approvalStatus', {}).get(class="hl-string">'code') == class="hl-string">'A_APPR': class=class="hl-string">"hl-comment"># Approved
                class=class="hl-string">"hl-comment"># Inferring Finance Approval from final approval date
                event_log.append({
                        **base_event,
                        class="hl-string">'ActivityName': class="hl-string">'Finance Approved',
                        class="hl-string">'EventTimestamp': details.get(class="hl-string">'approvedDate'),
                        class="hl-string">'ProcessorUserId': details.get(class="hl-string">'lastApprover', {}).get(class="hl-string">'id'),
                        class="hl-string">'ApproverId': details.get(class="hl-string">'lastApprover', {}).get(class="hl-string">'id'),
                        class="hl-string">'ExpenseCategory': None
                    })

            payment_status = details.get(class="hl-string">'paymentStatus', {}).get(class="hl-string">'code')
            if payment_status == class="hl-string">'P_PROC': class=class="hl-string">"hl-comment"># Processing Payment
                event_log.append({
                    **base_event,
                    class="hl-string">'ActivityName': class="hl-string">'Reimbursement Scheduled',
                    class="hl-string">'EventTimestamp': details.get(class="hl-string">'paymentProcessingDate'),
                    class="hl-string">'ProcessorUserId': class="hl-string">'System',
                    class="hl-string">'ApproverId': None,
                    class="hl-string">'ExpenseCategory': None
                })
            
            if payment_status == class="hl-string">'P_PAID': class=class="hl-string">"hl-comment"># Paid
                event_log.append({
                    **base_event,
                    class="hl-string">'ActivityName': class="hl-string">'Reimbursement Executed',
                    class="hl-string">'EventTimestamp': details.get(class="hl-string">'paidDate'),
                    class="hl-string">'ProcessorUserId': class="hl-string">'System',
                    class="hl-string">'ApproverId': None,
                    class="hl-string">'ExpenseCategory': None
                })

            if details.get(class="hl-string">'isPosted'):
                 event_log.append({
                    **base_event,
                    class="hl-string">'ActivityName': class="hl-string">'Accounting Posted',
                    class="hl-string">'EventTimestamp': details.get(class="hl-string">'postDate'),
                    class="hl-string">'ProcessorUserId': class="hl-string">'System',
                    class="hl-string">'ApproverId': None,
                    class="hl-string">'ExpenseCategory': None
                })
            
            class=class="hl-string">"hl-comment"># --- Other Inferred Activities --- #
            class=class="hl-string">"hl-comment"># Note: The following are examples and may need adjustment based on specific Concur configuration.
            if details.get(class="hl-string">'hasExceptions'):
                 event_log.append({
                    **base_event,
                    class="hl-string">'ActivityName': class="hl-string">'Policy Violation Flagged',
                    class="hl-string">'EventTimestamp': details.get(class="hl-string">'lastModifiedDate'), class=class="hl-string">"hl-comment"># Best guess timestamp
                    class="hl-string">'ProcessorUserId': class="hl-string">'System',
                    class="hl-string">'ApproverId': None,
                    class="hl-string">'ExpenseCategory': None
                })

            if details.get(class="hl-string">'workflow', {}).get(class="hl-string">'currentStep', {}).get(class="hl-string">'name', class="hl-string">'').lower().find(class="hl-string">'audit') != -1:
                 event_log.append({
                    **base_event,
                    class="hl-string">'ActivityName': class="hl-string">'Audit Review Required',
                    class="hl-string">'EventTimestamp': details.get(class="hl-string">'lastModifiedDate'),
                    class="hl-string">'ProcessorUserId': class="hl-string">'System',
                    class="hl-string">'ApproverId': None,
                    class="hl-string">'ExpenseCategory': None
                })

            class=class="hl-string">"hl-comment"># Placeholder for activities not directly available via standard fields
            class=class="hl-string">"hl-comment"># e.g., class="hl-string">'Receipts Image Attached', class="hl-string">'Finance Review Started' may require parsing specific audit trail descriptions

        next_page = reports_data.get(class="hl-string">'nextPage')
        if not next_page:
            break
        page_count += 1
    
    print(fclass="hl-string">'Extraction complete. Total events generated: {len(event_log)}')

    if event_log:
        df = pd.DataFrame(event_log)
        class=class="hl-string">"hl-comment"># Reorder columns for clarity
        cols = [class="hl-string">'ExpenseReportId', class="hl-string">'ActivityName', class="hl-string">'EventTimestamp', class="hl-string">'ProcessorUserId', 
                class="hl-string">'ReportTotalAmount', class="hl-string">'EmployeeDepartment', class="hl-string">'PolicyViolationFlag', class="hl-string">'ApproverId', 
                class="hl-string">'ExpenseCategory', class="hl-string">'SourceSystem', class="hl-string">'LastDataUpdate']
        df = df[cols]
        df.to_csv(class="hl-string">'concur_event_log.csv', index=False)
        print(class="hl-string">'Event log saved to concur_event_log.csv')
    else:
        print(class="hl-string">'No events were generated.')

if __name__ == class="hl-string">'__main__':
    main()