From 8dc3472797446589a38961fcda7cccb988a21244 Mon Sep 17 00:00:00 2001 From: Matthew Davidian <84351798+matthewdavidian@users.noreply.github.com> Date: Tue, 17 Feb 2026 16:07:05 -0800 Subject: [PATCH] Add DocuPipe async document pipeline interface --- AppSrc/cDocuPipeClient.pkg | 179 +++++++++++++++++++++ AppSrc/cDocuPipeDocumentService.pkg | 65 ++++++++ AppSrc/cDocuPipeJobService.pkg | 37 +++++ AppSrc/cDocuPipeStandardizationService.pkg | 74 +++++++++ AppSrc/docupipeai.h | 52 ++++++ AppSrc/docupipeai.pkg | 63 ++++++++ README.md | 55 +++++++ 7 files changed, 525 insertions(+) create mode 100644 AppSrc/cDocuPipeClient.pkg create mode 100644 AppSrc/cDocuPipeDocumentService.pkg create mode 100644 AppSrc/cDocuPipeJobService.pkg create mode 100644 AppSrc/cDocuPipeStandardizationService.pkg create mode 100644 AppSrc/docupipeai.h create mode 100644 AppSrc/docupipeai.pkg diff --git a/AppSrc/cDocuPipeClient.pkg b/AppSrc/cDocuPipeClient.pkg new file mode 100644 index 0000000..d073cd6 --- /dev/null +++ b/AppSrc/cDocuPipeClient.pkg @@ -0,0 +1,179 @@ +Use cdtJsonHttpTransfer.pkg +Use cBase64Attachment_mixin.pkg +Use cJsonObject.pkg +Use docupipeai.h + +Class cDocuPipeClient is a cdtJsonHttpTransfer + Import_Class_Protocol cBase64Attachment_mixin + + Procedure Construct_Object + Forward Send Construct_Object + + Send Define_cBase64Attachment_mixin + + Property String psApiKey "" + Property String psBaseUrl "https://app.docupipe.ai" + Property Integer piTimeoutSeconds 90 + Property Integer piRetryCount 1 + Property Boolean pbLogRawJson False + Property String psLastError "" + + Set peTransferFlags to ifSecure + Set piRemotePort to rpHttpSSL + End_Procedure + + Procedure Configure tDocuPipeConfig cfg + If (cfg.sApiKey<>'') Set psApiKey to cfg.sApiKey + If (cfg.sBaseUrl<>'') Set psBaseUrl to cfg.sBaseUrl + If (cfg.iTimeoutSeconds>0) Set piTimeoutSeconds to cfg.iTimeoutSeconds + If (cfg.iRetryCount>=0) Set piRetryCount to cfg.iRetryCount + Set pbLogRawJson to cfg.bLogRawJson + Send SetTimeout (piTimeoutSeconds(Self)) + End_Procedure + + Function HostFromBaseUrl Returns String + String sBase sHost + + Get psBaseUrl to sBase + Move (Replace("https://",sBase,"")) to sHost + Move (Replace("http://",sHost,"")) to sHost + If (Right(sHost,1)='/') Move (Left(sHost,Length(sHost)-1)) to sHost + + Function_Return sHost + End_Function + + Function NormalizePath String sPath Returns String + String sRet + Move sPath to sRet + While (Left(sRet,1)='/') + Move (Right(sRet,Length(sRet)-1)) to sRet + Loop + Function_Return sRet + End_Function + + Procedure AddStandardHeaders Boolean bJsonBody + Integer iRetVal + + Set pbClearHeaders to False + Send ClearHeaders + + Get AddHeader "Accept" "application/json" to iRetVal + Get AddHeader "X-API-Key" (psApiKey(Self)) to iRetVal + If (bJsonBody) Get AddHeader "Content-Type" "application/json" to iRetVal + End_Procedure + + Function ShouldRetry Boolean bTransferOk Integer iStatus Returns Boolean + Boolean bRetry + + Move False to bRetry + If (not(bTransferOk)) Move True to bRetry + Else If (iStatus=429) Move True to bRetry + Else If (iStatus>=500 and iStatus<=599) Move True to bRetry + + Function_Return bRetry + End_Function + + Function ExecuteJsonRequest String sVerb String sPath Handle hoRequest Returns tDocuPipeResult + tDocuPipeResult Result + Handle hoResponse + String sHost sRawJson sPathOnly + Boolean bTransferOk bJsonBody bRetry + Integer iAttempt iAttempts iStatus + + Move (Trim(sVerb)) to sVerb + Move (Uppercase(sVerb)) to sVerb + Move (NormalizePath(Self,sPath)) to sPathOnly + Move (HostFromBaseUrl(Self)) to sHost + Move (piRetryCount(Self)+1) to iAttempts + Move (hoRequest<>0) to bJsonBody + + If (psApiKey(Self)="") Begin + Move "DocuPipe API key is not configured." to Result.sError + Function_Return Result + End + + For iAttempt from 1 to iAttempts + Send AddStandardHeaders bJsonBody + + Move False to bTransferOk + If (sVerb="GET") Begin + Get HttpGetJson sHost sPathOnly (&bTransferOk) to hoResponse + End + Else Begin + Get HttpVerbJson sHost sPathOnly hoRequest sVerb (&bTransferOk) to hoResponse + End + + Get ResponseStatusCode to iStatus + Move iStatus to Result.iHttpStatus + + If (hoResponse) Begin + Move (Stringify(hoResponse)) to sRawJson + Send Destroy of hoResponse + Move 0 to hoResponse + End + Else Begin + Get DataReceived to sRawJson + End + + Move sRawJson to Result.sRawJson + Move (bTransferOk and iStatus>=200 and iStatus<300) to Result.bOk + + If (Result.bOk) Function_Return Result + + Move (ShouldRetry(Self,bTransferOk,iStatus)) to bRetry + If (not(bRetry) or iAttempt=iAttempts) Begin + If (sRawJson<>'') Move sRawJson to Result.sError + Else If (not(bTransferOk)) Move (TransferErrorDescription(Self)) to Result.sError + Else Move ('HTTP error '+String(iStatus)) to Result.sError + Function_Return Result + End + Loop + + Move "DocuPipe request failed after retries." to Result.sError + Function_Return Result + End_Function + + + Function ExtractJsonStringValue String sJson String sKey Returns String + Integer iKey iColon iStart iStop + String sNeedle sTail sValue + + Move ('"'+sKey+'"') to sNeedle + Move (Pos(sNeedle,sJson)) to iKey + If (iKey=0) Function_Return "" + + Move (Mid(sJson,Length(sJson)-iKey+1,iKey)) to sTail + Move (Pos(':',sTail)) to iColon + If (iColon=0) Function_Return "" + + Move (Mid(sTail,Length(sTail)-iColon+1,iColon)) to sTail + Move (Pos('"',sTail)) to iStart + If (iStart=0) Function_Return "" + + Move (Mid(sTail,Length(sTail)-iStart,iStart+1)) to sTail + Move (Pos('"',sTail)) to iStop + If (iStop=0) Function_Return "" + + Move (Left(sTail,iStop-1)) to sValue + Function_Return sValue + End_Function + + Function BuildDocumentFileRequest String sFilePath String sDataset String sWorkflowId Returns Handle + Handle hoReq + tAIAttachment Attachment + + Get Create (RefClass(cJsonObject)) to hoReq + Send InitializeJsonType of hoReq jsonTypeObject + + Get CreateAttachment sFilePath to Attachment + + Send SetMemberValue of hoReq "contentBase64" jsonTypeString (UCharArrayToString(Attachment.uBase64File)) + Send SetMemberValue of hoReq "mimeType" jsonTypeString Attachment.sMimeType + Send SetMemberValue of hoReq "filename" jsonTypeString sFilePath + + If (sDataset<>'') Send SetMemberValue of hoReq "dataset" jsonTypeString sDataset + If (sWorkflowId<>'') Send SetMemberValue of hoReq "workflowId" jsonTypeString sWorkflowId + + Function_Return hoReq + End_Function +End_Class diff --git a/AppSrc/cDocuPipeDocumentService.pkg b/AppSrc/cDocuPipeDocumentService.pkg new file mode 100644 index 0000000..cd9f0fd --- /dev/null +++ b/AppSrc/cDocuPipeDocumentService.pkg @@ -0,0 +1,65 @@ +Use cJsonObject.pkg +Use docupipeai.h +Use cDocuPipeClient.pkg + +Class cDocuPipeDocumentService is a cObject + + Function SubmitDocumentFromFile Handle hoClient String sFilePath String sDataset String sWorkflowId Returns tDocuPipeDocumentSubmitResponse + tDocuPipeDocumentSubmitResponse Response + tDocuPipeResult Result + Handle hoReq + + Get BuildDocumentFileRequest of hoClient sFilePath sDataset sWorkflowId to hoReq + Get ExecuteJsonRequest of hoClient "POST" "/document" hoReq to Result + Send Destroy of hoReq + + Move Result.bOk to Response.bOk + Move Result.iHttpStatus to Response.iHttpStatus + Move Result.sError to Response.sError + Move Result.sRawJson to Response.sRawJson + + If (Result.sRawJson<>'') Begin + Get ExtractJsonStringValue of hoClient Result.sRawJson "documentId" to Response.sDocumentId + If (Response.sDocumentId='') Get ExtractJsonStringValue of hoClient Result.sRawJson "document_id" to Response.sDocumentId + If (Response.sDocumentId='') Get ExtractJsonStringValue of hoClient Result.sRawJson "id" to Response.sDocumentId + End + + Function_Return Response + End_Function + + Function SubmitDocumentFromUrl Handle hoClient String sUrl String sDataset String sWorkflowId Returns tDocuPipeDocumentSubmitResponse + tDocuPipeDocumentSubmitResponse Response + tDocuPipeResult Result + Handle hoReq + + Get Create (RefClass(cJsonObject)) to hoReq + Send InitializeJsonType of hoReq jsonTypeObject + Send SetMemberValue of hoReq "url" jsonTypeString sUrl + If (sDataset<>'') Send SetMemberValue of hoReq "dataset" jsonTypeString sDataset + If (sWorkflowId<>'') Send SetMemberValue of hoReq "workflowId" jsonTypeString sWorkflowId + + Get ExecuteJsonRequest of hoClient "POST" "/document" hoReq to Result + Send Destroy of hoReq + + Move Result.bOk to Response.bOk + Move Result.iHttpStatus to Response.iHttpStatus + Move Result.sError to Response.sError + Move Result.sRawJson to Response.sRawJson + + If (Result.sRawJson<>'') Begin + Get ExtractJsonStringValue of hoClient Result.sRawJson "documentId" to Response.sDocumentId + If (Response.sDocumentId='') Get ExtractJsonStringValue of hoClient Result.sRawJson "document_id" to Response.sDocumentId + If (Response.sDocumentId='') Get ExtractJsonStringValue of hoClient Result.sRawJson "id" to Response.sDocumentId + End + + Function_Return Response + End_Function + + Function GetDocument Handle hoClient String sDocumentId Returns tDocuPipeResult + tDocuPipeResult Result + + Get ExecuteJsonRequest of hoClient "GET" ("/document/"+sDocumentId) 0 to Result + Function_Return Result + End_Function + +End_Class diff --git a/AppSrc/cDocuPipeJobService.pkg b/AppSrc/cDocuPipeJobService.pkg new file mode 100644 index 0000000..ba2adcc --- /dev/null +++ b/AppSrc/cDocuPipeJobService.pkg @@ -0,0 +1,37 @@ +Use docupipeai.h +Use cDocuPipeClient.pkg + +Class cDocuPipeJobService is a cObject + + Function GetJob Handle hoClient String sJobId Returns tDocuPipeJobResponse + tDocuPipeJobResponse Response + tDocuPipeResult Result + + Get ExecuteJsonRequest of hoClient "GET" ("/job/"+sJobId) 0 to Result + + Move Result.bOk to Response.bOk + Move Result.iHttpStatus to Response.iHttpStatus + Move Result.sError to Response.sError + Move Result.sRawJson to Response.sRawJson + Move sJobId to Response.sJobId + + If (Result.sRawJson<>'') Begin + Get ExtractJsonStringValue of hoClient Result.sRawJson "jobId" to Response.sJobId + If (Response.sJobId='') Get ExtractJsonStringValue of hoClient Result.sRawJson "job_id" to Response.sJobId + + Get ExtractJsonStringValue of hoClient Result.sRawJson "status" to Response.sStatus + Get ExtractJsonStringValue of hoClient Result.sRawJson "standardizationId" to Response.sStandardizationId + If (Response.sStandardizationId='') Get ExtractJsonStringValue of hoClient Result.sRawJson "standardization_id" to Response.sStandardizationId + End + + Function_Return Response + End_Function + + Function IsTerminalJobStatus String sStatus Returns Boolean + String sCompare + + Move (Lowercase(Trim(sStatus))) to sCompare + Function_Return (sCompare="completed" or sCompare="complete" or sCompare="failed" or sCompare="error" or sCompare="cancelled" or sCompare="canceled") + End_Function + +End_Class diff --git a/AppSrc/cDocuPipeStandardizationService.pkg b/AppSrc/cDocuPipeStandardizationService.pkg new file mode 100644 index 0000000..310761c --- /dev/null +++ b/AppSrc/cDocuPipeStandardizationService.pkg @@ -0,0 +1,74 @@ +Use cJsonObject.pkg +Use docupipeai.h +Use cDocuPipeClient.pkg + +Class cDocuPipeStandardizationService is a cObject + + Function Standardize Handle hoClient String[] aDocumentIds String sSchemaId String sGuidelines String sDisplayMode String sSplitMode String sEffortLevel Returns tDocuPipeStandardizeResponse + tDocuPipeStandardizeResponse Response + tDocuPipeResult Result + Handle hoReq hoIds hoId + Integer i iCount + + Get Create (RefClass(cJsonObject)) to hoReq + Send InitializeJsonType of hoReq jsonTypeObject + + Get Create (RefClass(cJsonObject)) to hoIds + Send InitializeJsonType of hoIds jsonTypeArray + + Move (SizeOfArray(aDocumentIds)) to iCount + For i from 0 to (iCount-1) + Get Create (RefClass(cJsonObject)) to hoId + Send InitializeJsonType of hoId jsonTypeString + Send SetStringValue of hoId aDocumentIds[i] + Send AddMember of hoIds hoId + Send Destroy of hoId + Loop + + Send SetMember of hoReq "documentIds" hoIds + Send Destroy of hoIds + + If (sSchemaId<>'') Send SetMemberValue of hoReq "schemaId" jsonTypeString sSchemaId + If (sGuidelines<>'') Send SetMemberValue of hoReq "guidelines" jsonTypeString sGuidelines + If (sDisplayMode<>'') Send SetMemberValue of hoReq "displayMode" jsonTypeString sDisplayMode + If (sSplitMode<>'') Send SetMemberValue of hoReq "splitMode" jsonTypeString sSplitMode + If (sEffortLevel<>'') Send SetMemberValue of hoReq "effortLevel" jsonTypeString sEffortLevel + + Get ExecuteJsonRequest of hoClient "POST" "/v2/standardize/batch" hoReq to Result + Send Destroy of hoReq + + Move Result.bOk to Response.bOk + Move Result.iHttpStatus to Response.iHttpStatus + Move Result.sError to Response.sError + Move Result.sRawJson to Response.sRawJson + + If (Result.sRawJson<>'') Begin + Get ExtractJsonStringValue of hoClient Result.sRawJson "jobId" to Response.sJobId + If (Response.sJobId='') Get ExtractJsonStringValue of hoClient Result.sRawJson "job_id" to Response.sJobId + If (Response.sJobId='') Get ExtractJsonStringValue of hoClient Result.sRawJson "id" to Response.sJobId + End + + Function_Return Response + End_Function + + Function GetStandardization Handle hoClient String sStandardizationId Returns tDocuPipeStandardizationGetResponse + tDocuPipeStandardizationGetResponse Response + tDocuPipeResult Result + + Get ExecuteJsonRequest of hoClient "GET" ("/standardization/"+sStandardizationId) 0 to Result + + Move Result.bOk to Response.bOk + Move Result.iHttpStatus to Response.iHttpStatus + Move Result.sError to Response.sError + Move Result.sRawJson to Response.sRawJson + Move sStandardizationId to Response.sStandardizationId + + If (Result.sRawJson<>'') Begin + Get ExtractJsonStringValue of hoClient Result.sRawJson "standardizationId" to Response.sStandardizationId + If (Response.sStandardizationId='') Get ExtractJsonStringValue of hoClient Result.sRawJson "standardization_id" to Response.sStandardizationId + Move Result.sRawJson to Response.sDataJson + End + + Function_Return Response + End_Function +End_Class diff --git a/AppSrc/docupipeai.h b/AppSrc/docupipeai.h new file mode 100644 index 0000000..4fe3f24 --- /dev/null +++ b/AppSrc/docupipeai.h @@ -0,0 +1,52 @@ +Use Windows.pkg + +Struct tDocuPipeConfig + String sApiKey + String sBaseUrl + Integer iTimeoutSeconds + Integer iRetryCount + Boolean bLogRawJson +End_Struct + +Struct tDocuPipeResult + Boolean bOk + Integer iHttpStatus + String sError + String sRawJson + String sRequestId +End_Struct + +Struct tDocuPipeDocumentSubmitResponse + Boolean bOk + Integer iHttpStatus + String sError + String sDocumentId + String sRawJson +End_Struct + +Struct tDocuPipeStandardizeResponse + Boolean bOk + Integer iHttpStatus + String sError + String sJobId + String sRawJson +End_Struct + +Struct tDocuPipeJobResponse + Boolean bOk + Integer iHttpStatus + String sError + String sJobId + String sStatus + String sStandardizationId + String sRawJson +End_Struct + +Struct tDocuPipeStandardizationGetResponse + Boolean bOk + Integer iHttpStatus + String sError + String sStandardizationId + String sRawJson + String sDataJson +End_Struct diff --git a/AppSrc/docupipeai.pkg b/AppSrc/docupipeai.pkg new file mode 100644 index 0000000..59688db --- /dev/null +++ b/AppSrc/docupipeai.pkg @@ -0,0 +1,63 @@ +Use docupipeai.h +Use cDocuPipeClient.pkg +Use cDocuPipeDocumentService.pkg +Use cDocuPipeStandardizationService.pkg +Use cDocuPipeJobService.pkg + +Class cDocuPipeInterface is a cObject + + Procedure Construct_Object + Forward Send Construct_Object + + Object oDocuPipeClient is a cDocuPipeClient + End_Object + + Object oDocuPipeDocumentService is a cDocuPipeDocumentService + End_Object + + Object oDocuPipeStandardizationService is a cDocuPipeStandardizationService + End_Object + + Object oDocuPipeJobService is a cDocuPipeJobService + End_Object + End_Procedure + + Procedure Configure tDocuPipeConfig cfg + Send Configure of (oDocuPipeClient(Self)) cfg + End_Procedure + + Function SubmitDocumentFromFile String sFilePath String sDataset String sWorkflowId Returns tDocuPipeDocumentSubmitResponse + Function_Return (SubmitDocumentFromFile(oDocuPipeDocumentService(Self),oDocuPipeClient(Self),sFilePath,sDataset,sWorkflowId)) + End_Function + + Function SubmitDocumentFromUrl String sUrl String sDataset String sWorkflowId Returns tDocuPipeDocumentSubmitResponse + Function_Return (SubmitDocumentFromUrl(oDocuPipeDocumentService(Self),oDocuPipeClient(Self),sUrl,sDataset,sWorkflowId)) + End_Function + + Function GetDocument String sDocumentId Returns tDocuPipeResult + Function_Return (GetDocument(oDocuPipeDocumentService(Self),oDocuPipeClient(Self),sDocumentId)) + End_Function + + Function Standardize String[] aDocumentIds String sSchemaId String sGuidelines String sDisplayMode String sSplitMode String sEffortLevel Returns tDocuPipeStandardizeResponse + Function_Return (Standardize(oDocuPipeStandardizationService(Self),oDocuPipeClient(Self),aDocumentIds,sSchemaId,sGuidelines,sDisplayMode,sSplitMode,sEffortLevel)) + End_Function + + Function GetJob String sJobId Returns tDocuPipeJobResponse + Function_Return (GetJob(oDocuPipeJobService(Self),oDocuPipeClient(Self),sJobId)) + End_Function + + Function GetStandardization String sStandardizationId Returns tDocuPipeStandardizationGetResponse + Function_Return (GetStandardization(oDocuPipeStandardizationService(Self),oDocuPipeClient(Self),sStandardizationId)) + End_Function + + Function IsTerminalJobStatus String sStatus Returns Boolean + Function_Return (IsTerminalJobStatus(oDocuPipeJobService(Self),sStatus)) + End_Function + +End_Class + +Global_Variable Handle ghoDocuPipe + +Object oDocuPipe is a cDocuPipeInterface + Move Self to ghoDocuPipe +End_Object diff --git a/README.md b/README.md index 6f38239..8f6f808 100644 --- a/README.md +++ b/README.md @@ -157,3 +157,58 @@ There are two views, one that allows you to submit a request to any of the four [ ] On the Request side, support other parameters besides the max tokens, model id (e.g. Temperature, number of completions, top_p, etc) [ ] REplace backup of database with a script to create tables/etc. Matt Davidian August 2025 + + +# DOCUPIPE PIPELINE INTERFACE +DocuPipe support is implemented as a parallel **document pipeline** interface (not part of `cAiInterface`). + +Primary entry point: +- `Use docupipeai.pkg` +- Global object handle: `ghoDocuPipe` + +Main async-first calls: +- `Configure tDocuPipeConfig cfg` +- `SubmitDocumentFromFile ...` / `SubmitDocumentFromUrl ...` +- `Standardize ...` +- `GetJob ...` +- `GetStandardization ...` +- `GetDocument ...` + +Example polling flow: + +```dataflex +Use docupipeai.pkg + +Procedure RunDocuPipeExample + tDocuPipeConfig cfg + tDocuPipeDocumentSubmitResponse submitResp + tDocuPipeStandardizeResponse standardizeResp + tDocuPipeJobResponse jobResp + tDocuPipeStandardizationGetResponse stdResp + String[] aDocumentIds + + Move (EnvironmentVariable(ghoEnvironment,"docupipe-api-key")) to cfg.sApiKey + Move "https://app.docupipe.ai" to cfg.sBaseUrl + Move 90 to cfg.iTimeoutSeconds + Move 1 to cfg.iRetryCount + Send Configure of ghoDocuPipe cfg + + Get SubmitDocumentFromUrl of ghoDocuPipe "https://example.com/invoice.pdf" "" "" to submitResp + If (not(submitResp.bOk)) Procedure_Return + + Move submitResp.sDocumentId to aDocumentIds[0] + + Get Standardize of ghoDocuPipe aDocumentIds "" "" "" "" "" to standardizeResp + If (not(standardizeResp.bOk)) Procedure_Return + + Repeat + Get GetJob of ghoDocuPipe standardizeResp.sJobId to jobResp + If (IsTerminalJobStatus(ghoDocuPipe,jobResp.sStatus)) Break + Sleep 2 + Until False + + If (Lowercase(jobResp.sStatus)="completed") Begin + Get GetStandardization of ghoDocuPipe jobResp.sStandardizationId to stdResp + End +End_Procedure +```