Add Activepieces integration for workflow automation

- Add Activepieces fork with SmoothSchedule custom piece
- Create integrations app with Activepieces service layer
- Add embed token endpoint for iframe integration
- Create Automations page with embedded workflow builder
- Add sidebar visibility fix for embed mode
- Add list inactive customers endpoint to Public API
- Include SmoothSchedule triggers: event created/updated/cancelled
- Include SmoothSchedule actions: create/update/cancel events, list resources/services/customers

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

activepieces-fork/packages/engine/src/lib/core/code/code-sandbox-common.ts

@@ -0,0 +1,47 @@
+export type CodeModule = {
+    code(input: unknown): Promise<unknown>
+}
+
+export type CodeSandbox = {
+    /**
+     * Executes a {@link CodeModule}.
+     */
+    runCodeModule(params: RunCodeModuleParams): Promise<unknown>
+
+    /**
+     * Executes a script.
+     */
+    runScript(params: RunScriptParams): Promise<unknown>
+}
+
+type RunCodeModuleParams = {
+    /**
+     * The {@link CodeModule} to execute.
+     */
+    codeModule: CodeModule
+
+    /**
+     * The inputs that are passed to the {@link CodeModule}.
+     */
+    inputs: Record<string, unknown>
+}
+
+type RunScriptParams = {
+    /**
+     * A serialized script that will be executed in the sandbox.
+     * The script can either be sync or async.
+     */
+    script: string
+
+    /**
+     * A key-value map of variables available to the script during execution.
+     */
+    scriptContext: Record<string, unknown>
+
+    /**
+     * A key-value map of functions that are available to the script during execution.
+     */
+    // eslint-disable-next-line @typescript-eslint/ban-types
+    functions: Record<string, Function>
+
+}

activepieces-fork/packages/engine/src/lib/core/code/code-sandbox.ts

@@ -0,0 +1,39 @@
+import { EngineGenericError, ExecutionMode, isNil } from '@activepieces/shared'
+import { CodeSandbox } from '../../core/code/code-sandbox-common'
+export const EXECUTION_MODE = (process.env.AP_EXECUTION_MODE as ExecutionMode)
+
+const loadNoOpCodeSandbox = async (): Promise<CodeSandbox> => {
+    const noOpCodeSandboxModule = await import('./no-op-code-sandbox')
+    return noOpCodeSandboxModule.noOpCodeSandbox
+}
+
+const loadV8IsolateSandbox = async (): Promise<CodeSandbox> => {
+    const v8IsolateCodeSandboxModule = await import('./v8-isolate-code-sandbox')
+    return v8IsolateCodeSandboxModule.v8IsolateCodeSandbox
+}
+
+const loadCodeSandbox = async (): Promise<CodeSandbox> => {
+    const loaders = {
+        [ExecutionMode.UNSANDBOXED]: loadNoOpCodeSandbox,
+        [ExecutionMode.SANDBOX_PROCESS]: loadNoOpCodeSandbox,
+        [ExecutionMode.SANDBOX_CODE_ONLY]: loadV8IsolateSandbox,
+        [ExecutionMode.SANDBOX_CODE_AND_PROCESS]: loadV8IsolateSandbox,
+    }
+
+    if (isNil(EXECUTION_MODE)) {
+        throw new EngineGenericError('ExecutionModeNotSetError', 'AP_EXECUTION_MODE environment variable is not set')
+    }
+    
+    const loader = loaders[EXECUTION_MODE]
+    return loader()
+}
+
+let instance: CodeSandbox | null = null
+
+export const initCodeSandbox = async (): Promise<CodeSandbox> => {
+    if (instance === null) {
+        instance = await loadCodeSandbox()
+    }
+
+    return instance
+}

activepieces-fork/packages/engine/src/lib/core/code/no-op-code-sandbox.ts

@@ -0,0 +1,22 @@
+import { CodeSandbox } from '../../core/code/code-sandbox-common'
+
+/**
+ * Runs code without a sandbox.
+ */
+export const noOpCodeSandbox: CodeSandbox = {
+    async runCodeModule({ codeModule, inputs }) {
+        return codeModule.code(inputs)
+    },
+
+    async runScript({ script, scriptContext, functions }) {
+        const newContext = {
+            ...scriptContext,
+            ...functions,
+        }   
+        const params = Object.keys(newContext)
+        const args = Object.values(newContext)
+        const body = `return (${script})`
+        const fn = Function(...params, body)
+        return fn(...args)
+    },
+}

activepieces-fork/packages/engine/src/lib/core/code/v8-isolate-code-sandbox.ts

@@ -0,0 +1,111 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { CodeModule, CodeSandbox } from '../../core/code/code-sandbox-common'
+
+const ONE_HUNDRED_TWENTY_EIGHT_MEGABYTES = 128
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+// Check this https://github.com/laverdet/isolated-vm/issues/258#issuecomment-2134341086
+let ivmCache: any
+const getIvm = () => {
+    if (!ivmCache) {
+        ivmCache = require('isolated-vm')
+    }
+    return ivmCache as typeof import('isolated-vm')
+}
+
+/**
+ * Runs code in a V8 Isolate sandbox
+ */
+export const v8IsolateCodeSandbox: CodeSandbox = {
+    async runCodeModule({ codeModule, inputs }) {
+        const ivm = getIvm()
+        const isolate = new ivm.Isolate({ memoryLimit: ONE_HUNDRED_TWENTY_EIGHT_MEGABYTES })
+
+        try {
+            const isolateContext = await initIsolateContext({
+                isolate,
+                codeContext: {
+                    inputs,
+                },
+            })
+
+            const serializedCodeModule = serializeCodeModule(codeModule)
+
+            return await executeIsolate({
+                isolate,
+                isolateContext,
+                code: serializedCodeModule,
+            })
+        }
+        finally {
+            isolate.dispose()
+        }
+    },
+
+    async runScript({ script, scriptContext, functions }) {
+        const ivm = getIvm()
+        const isolate = new ivm.Isolate({ memoryLimit: ONE_HUNDRED_TWENTY_EIGHT_MEGABYTES })
+
+        try {
+            // It is to avoid strucutedClone issue of proxy objects / functions, It will throw cannot be cloned error.
+            const isolateContext = await initIsolateContext({
+                isolate,
+                codeContext: JSON.parse(JSON.stringify(scriptContext)),
+            })
+
+            const serializedFunctions = Object.entries(functions).map(([key, value]) => `const ${key} = ${value.toString()};`).join('\n')
+            const scriptWithFunctions = `${serializedFunctions}\n${script}`
+
+            return await executeIsolate({
+                isolate,
+                isolateContext,
+                code: scriptWithFunctions,
+            })
+        }
+        finally {
+            isolate.dispose()
+        }
+    },
+}
+
+const initIsolateContext = async ({ isolate, codeContext }: InitContextParams): Promise<any> => {
+    const isolateContext = await isolate.createContext()
+    const ivm = getIvm()
+    for (const [key, value] of Object.entries(codeContext)) {
+        await isolateContext.global.set(key, new ivm.ExternalCopy(value).copyInto())
+    }
+
+    return isolateContext
+}
+
+const executeIsolate = async ({ isolate, isolateContext, code }: ExecuteIsolateParams): Promise<unknown> => {
+    const isolateScript = await isolate.compileScript(code)
+
+    const outRef = await isolateScript.run(isolateContext, {
+        reference: true,
+        promise: true,
+    })
+
+    return outRef.copy()
+}
+
+const serializeCodeModule = (codeModule: CodeModule): string => {
+    const serializedCodeFunction = Object.keys(codeModule)
+        .reduce((acc, key) => 
+            acc + `const ${key} = ${(codeModule as any)[key].toString()};`, 
+        '')
+
+    // replace the exports.function_name with function_name
+    return serializedCodeFunction.replace(/\(0, exports\.(\w+)\)/g, '$1') + 'code(inputs);'
+}
+
+type InitContextParams = {
+    isolate: any
+    codeContext: Record<string, unknown>
+}
+
+type ExecuteIsolateParams = {
+    isolate: any
+    isolateContext: unknown
+    code: string
+}

activepieces-fork/packages/engine/src/lib/handler/base-executor.ts

@@ -0,0 +1,13 @@
+import { FlowAction } from '@activepieces/shared'
+import { EngineConstants } from './context/engine-constants'
+import { FlowExecutorContext } from './context/flow-execution-context'
+
+export type ActionHandler<T extends FlowAction> = (request: { action: T, executionState: FlowExecutorContext, constants: EngineConstants }) => Promise<FlowExecutorContext>
+
+export type BaseExecutor<T extends FlowAction> = {
+    handle(request: {
+        action: T
+        executionState: FlowExecutorContext
+        constants: EngineConstants
+    }): Promise<FlowExecutorContext>
+}

activepieces-fork/packages/engine/src/lib/handler/code-executor.ts

@@ -0,0 +1,77 @@
+import path from 'path'
+import importFresh from '@activepieces/import-fresh-webpack'
+import { LATEST_CONTEXT_VERSION } from '@activepieces/pieces-framework'
+import { CodeAction, EngineGenericError, FlowActionType, FlowRunStatus, GenericStepOutput, isNil, StepOutputStatus } from '@activepieces/shared'
+import { initCodeSandbox } from '../core/code/code-sandbox'
+import { CodeModule } from '../core/code/code-sandbox-common'
+import { continueIfFailureHandler, runWithExponentialBackoff } from '../helper/error-handling'
+import { progressService } from '../services/progress.service'
+import { utils } from '../utils'
+import { ActionHandler, BaseExecutor } from './base-executor'
+
+export const codeExecutor: BaseExecutor<CodeAction> = {
+    async handle({
+        action,
+        executionState,
+        constants,
+    }) {
+        if (executionState.isCompleted({ stepName: action.name })) {
+            return executionState
+        }
+        const resultExecution = await runWithExponentialBackoff(executionState, action, constants, executeAction)
+        return continueIfFailureHandler(resultExecution, action, constants)
+    },
+}
+
+const executeAction: ActionHandler<CodeAction> = async ({ action, executionState, constants }) => {
+    const stepStartTime = performance.now() 
+    const { censoredInput, resolvedInput } = await constants.getPropsResolver(LATEST_CONTEXT_VERSION).resolve<Record<string, unknown>>({
+        unresolvedInput: action.settings.input,
+        executionState,
+    })
+
+    const stepOutput = GenericStepOutput.create({
+        input: censoredInput,
+        type: FlowActionType.CODE,
+        status: StepOutputStatus.RUNNING,
+    })
+    
+    const { data: executionStateResult, error: executionStateError } = await utils.tryCatchAndThrowOnEngineError((async () => {
+        await progressService.sendUpdate({
+            engineConstants: constants,
+            flowExecutorContext: executionState.upsertStep(action.name, stepOutput),
+        })
+    
+        if (isNil(constants.runEnvironment)) {
+            throw new EngineGenericError('RunEnvironmentNotSetError', 'Run environment is not set')
+        }
+
+        const artifactPath = path.resolve(`${constants.baseCodeDirectory}/${constants.flowVersionId}/${action.name}/index.js`)
+        const codeModule: CodeModule = await importFresh(artifactPath)
+        const codeSandbox = await initCodeSandbox()
+    
+        const output = await codeSandbox.runCodeModule({
+            codeModule,
+            inputs: resolvedInput,
+        })
+    
+        return executionState.upsertStep(action.name, stepOutput.setOutput(output).setStatus(StepOutputStatus.SUCCEEDED).setDuration(performance.now() - stepStartTime)).incrementStepsExecuted()
+    }))
+
+    if (executionStateError) {
+        const failedStepOutput = stepOutput
+            .setStatus(StepOutputStatus.FAILED)
+            .setErrorMessage(utils.formatError(executionStateError))
+            .setDuration(performance.now() - stepStartTime)
+
+        return executionState
+            .upsertStep(action.name, failedStepOutput)
+            .setVerdict({ status: FlowRunStatus.FAILED, failedStep: {
+                name: action.name,
+                displayName: action.displayName,
+                message: utils.formatError(executionStateError),
+            } })
+    }
+
+    return executionStateResult
+}

activepieces-fork/packages/engine/src/lib/handler/context/engine-constants.ts

@@ -0,0 +1,241 @@
+import { ContextVersion } from '@activepieces/pieces-framework'
+import { DEFAULT_MCP_DATA, EngineGenericError, ExecuteFlowOperation, ExecutePropsOptions, ExecuteToolOperation, ExecuteTriggerOperation, ExecutionType, FlowVersionState, PlatformId, ProgressUpdateType, Project, ProjectId, ResumePayload, RunEnvironment, TriggerHookType } from '@activepieces/shared'
+import { createPropsResolver, PropsResolver } from '../../variables/props-resolver'
+
+type RetryConstants = {
+    maxAttempts: number
+    retryExponential: number
+    retryInterval: number
+}
+
+type EngineConstantsParams = {
+    flowId: string
+    flowVersionId: string
+    flowVersionState: FlowVersionState
+    triggerPieceName: string
+    flowRunId: string
+    publicApiUrl: string
+    internalApiUrl: string
+    retryConstants: RetryConstants
+    engineToken: string
+    projectId: ProjectId
+    progressUpdateType: ProgressUpdateType
+    serverHandlerId: string | null
+    httpRequestId: string | null
+    resumePayload?: ResumePayload
+    runEnvironment?: RunEnvironment
+    stepNameToTest?: string
+    logsUploadUrl?: string
+    logsFileId?: string
+    timeoutInSeconds: number
+    platformId: PlatformId
+}
+
+const DEFAULT_RETRY_CONSTANTS: RetryConstants = {
+    maxAttempts: 4,
+    retryExponential: 2,
+    retryInterval: 2000,
+}
+
+const DEFAULT_TRIGGER_EXECUTION = 'execute-trigger'
+const DEFAULT_EXECUTE_PROPERTY = 'execute-property'
+
+export class EngineConstants {
+    public static readonly BASE_CODE_DIRECTORY = process.env.AP_BASE_CODE_DIRECTORY ?? './codes'
+    public static readonly INPUT_FILE = './input.json'
+    public static readonly OUTPUT_FILE = './output.json'
+    public static readonly DEV_PIECES = process.env.AP_DEV_PIECES?.split(',') ?? []
+    public static readonly TEST_MODE = process.env.AP_TEST_MODE === 'true'
+
+    public readonly platformId: string
+    public readonly timeoutInSeconds: number
+    public readonly flowId: string
+    public readonly flowVersionId: string
+    public readonly flowVersionState: FlowVersionState
+    public readonly triggerPieceName: string
+    public readonly flowRunId: string
+    public readonly publicApiUrl: string
+    public readonly internalApiUrl: string
+    public readonly retryConstants: RetryConstants
+    public readonly engineToken: string
+    public readonly projectId: ProjectId
+    public readonly progressUpdateType: ProgressUpdateType
+    public readonly serverHandlerId: string | null
+    public readonly httpRequestId: string | null
+    public readonly resumePayload?: ResumePayload
+    public readonly runEnvironment?: RunEnvironment
+    public readonly stepNameToTest?: string
+    public readonly logsUploadUrl?: string
+    public readonly logsFileId?: string
+    private project: Project | null = null
+
+    public get isRunningApTests(): boolean {
+        return EngineConstants.TEST_MODE
+    }
+
+    public get baseCodeDirectory(): string {
+        return EngineConstants.BASE_CODE_DIRECTORY
+    }
+
+    public get devPieces(): string[] {
+        return EngineConstants.DEV_PIECES
+    }
+
+    public constructor(params: EngineConstantsParams) {
+        if (!params.publicApiUrl.endsWith('/api/')) {
+            throw new EngineGenericError('PublicUrlNotEndsWithSlashError', `Public URL must end with a slash, got: ${params.publicApiUrl}`)
+        }
+        if (!params.internalApiUrl.endsWith('/')) {
+            throw new EngineGenericError('InternalApiUrlNotEndsWithSlashError', `Internal API URL must end with a slash, got: ${params.internalApiUrl}`)
+        }
+
+        this.flowId = params.flowId
+        this.flowVersionId = params.flowVersionId
+        this.flowVersionState = params.flowVersionState
+        this.flowRunId = params.flowRunId
+        this.publicApiUrl = params.publicApiUrl
+        this.internalApiUrl = params.internalApiUrl
+        this.retryConstants = params.retryConstants
+        this.triggerPieceName = params.triggerPieceName
+        this.engineToken = params.engineToken
+        this.projectId = params.projectId
+        this.progressUpdateType = params.progressUpdateType
+        this.serverHandlerId = params.serverHandlerId
+        this.httpRequestId = params.httpRequestId
+        this.resumePayload = params.resumePayload
+        this.runEnvironment = params.runEnvironment
+        this.stepNameToTest = params.stepNameToTest
+        this.logsUploadUrl = params.logsUploadUrl
+        this.logsFileId = params.logsFileId
+        this.platformId = params.platformId
+        this.timeoutInSeconds = params.timeoutInSeconds
+    }
+  
+    public static fromExecuteFlowInput(input: ExecuteFlowOperation): EngineConstants {
+        return new EngineConstants({
+            flowId: input.flowVersion.flowId,
+            flowVersionId: input.flowVersion.id,
+            flowVersionState: input.flowVersion.state,
+            triggerPieceName: input.flowVersion.trigger.settings.pieceName,
+            flowRunId: input.flowRunId,
+            publicApiUrl: input.publicApiUrl,
+            internalApiUrl: input.internalApiUrl,
+            retryConstants: DEFAULT_RETRY_CONSTANTS,
+            engineToken: input.engineToken,
+            projectId: input.projectId,
+            progressUpdateType: input.progressUpdateType,
+            serverHandlerId: input.serverHandlerId ?? null,
+            httpRequestId: input.httpRequestId ?? null,
+            resumePayload: input.executionType === ExecutionType.RESUME ? input.resumePayload : undefined,
+            runEnvironment: input.runEnvironment,
+            stepNameToTest: input.stepNameToTest ?? undefined,
+            logsUploadUrl: input.logsUploadUrl, 
+            logsFileId: input.logsFileId,
+            timeoutInSeconds: input.timeoutInSeconds,
+            platformId: input.platformId,
+        })
+    }
+
+    public static fromExecuteActionInput(input: ExecuteToolOperation): EngineConstants {
+        return new EngineConstants({
+            flowId: DEFAULT_MCP_DATA.flowId,
+            flowVersionId: DEFAULT_MCP_DATA.flowVersionId,
+            flowVersionState: DEFAULT_MCP_DATA.flowVersionState,
+            triggerPieceName: DEFAULT_MCP_DATA.triggerPieceName,
+            flowRunId: DEFAULT_MCP_DATA.flowRunId,
+            publicApiUrl: input.publicApiUrl,
+            internalApiUrl: addTrailingSlashIfMissing(input.internalApiUrl),
+            retryConstants: DEFAULT_RETRY_CONSTANTS,
+            engineToken: input.engineToken,
+            projectId: input.projectId,
+            progressUpdateType: ProgressUpdateType.NONE,
+            serverHandlerId: null,
+            httpRequestId: null,
+            resumePayload: undefined,
+            runEnvironment: undefined,
+            stepNameToTest: undefined,
+            timeoutInSeconds: input.timeoutInSeconds,
+            platformId: input.platformId,
+        })
+    }
+
+    public static fromExecutePropertyInput(input: Omit<ExecutePropsOptions, 'piece'> & { pieceName: string, pieceVersion: string }): EngineConstants {
+        return new EngineConstants({
+            flowId: input.flowVersion?.flowId ?? DEFAULT_MCP_DATA.flowId,
+            flowVersionId: input.flowVersion?.id ?? DEFAULT_MCP_DATA.flowVersionId,
+            flowVersionState: input.flowVersion?.state ?? DEFAULT_MCP_DATA.flowVersionState,
+            triggerPieceName: input.flowVersion?.trigger?.settings.pieceName ?? DEFAULT_MCP_DATA.triggerPieceName,
+            flowRunId: DEFAULT_EXECUTE_PROPERTY,
+            publicApiUrl: input.publicApiUrl,
+            internalApiUrl: addTrailingSlashIfMissing(input.internalApiUrl),
+            retryConstants: DEFAULT_RETRY_CONSTANTS,
+            engineToken: input.engineToken,
+            projectId: input.projectId,
+            progressUpdateType: ProgressUpdateType.NONE,
+            serverHandlerId: null,
+            httpRequestId: null,
+            resumePayload: undefined,
+            runEnvironment: undefined,
+            stepNameToTest: undefined,
+            timeoutInSeconds: input.timeoutInSeconds,
+            platformId: input.platformId,
+        })
+    }
+
+    public static fromExecuteTriggerInput(input: ExecuteTriggerOperation<TriggerHookType>): EngineConstants {
+        return new EngineConstants({
+            flowId: input.flowVersion.flowId,
+            flowVersionId: input.flowVersion.id,
+            flowVersionState: input.flowVersion.state,
+            triggerPieceName: input.flowVersion.trigger.settings.pieceName,
+            flowRunId: DEFAULT_TRIGGER_EXECUTION,
+            publicApiUrl: input.publicApiUrl,
+            internalApiUrl: addTrailingSlashIfMissing(input.internalApiUrl),
+            retryConstants: DEFAULT_RETRY_CONSTANTS,
+            engineToken: input.engineToken,
+            projectId: input.projectId,
+            progressUpdateType: ProgressUpdateType.NONE,
+            serverHandlerId: null,
+            httpRequestId: null,
+            resumePayload: undefined,
+            runEnvironment: undefined,
+            stepNameToTest: undefined,
+            timeoutInSeconds: input.timeoutInSeconds,
+            platformId: input.platformId,
+        })
+    }
+    public getPropsResolver(contextVersion: ContextVersion | undefined): PropsResolver {
+        return createPropsResolver({
+            projectId: this.projectId,
+            engineToken: this.engineToken,
+            apiUrl: this.internalApiUrl,
+            contextVersion,
+        })
+    }
+    private async getProject(): Promise<Project> {
+        if (this.project) {
+            return this.project
+        }
+
+        const getWorkerProjectEndpoint = `${this.internalApiUrl}v1/worker/project`
+
+        const response = await fetch(getWorkerProjectEndpoint, {
+            headers: {
+                Authorization: `Bearer ${this.engineToken}`,
+            },
+        })
+
+        this.project = await response.json() as Project
+        return this.project
+    }
+
+    public externalProjectId = async (): Promise<string | undefined> => {
+        const project = await this.getProject()
+        return project.externalId
+    }
+}
+
+
+const addTrailingSlashIfMissing = (url: string): string => {
+    return url.endsWith('/') ? url : url + '/'
+}

activepieces-fork/packages/engine/src/lib/handler/context/flow-execution-context.ts

@@ -0,0 +1,213 @@
+import { assertEqual, EngineGenericError, FailedStep, FlowActionType, FlowRunStatus, GenericStepOutput, isNil, LoopStepOutput, LoopStepResult, PauseMetadata, PauseType, RespondResponse, StepOutput, StepOutputStatus } from '@activepieces/shared'
+import dayjs from 'dayjs'
+import { nanoid } from 'nanoid'
+import { loggingUtils } from '../../helper/logging-utils'
+import { StepExecutionPath } from './step-execution-path'
+
+
+export type FlowVerdict = {
+    status: FlowRunStatus.PAUSED
+    pauseMetadata: PauseMetadata
+} | {
+    status: FlowRunStatus.SUCCEEDED
+    stopResponse: RespondResponse | undefined
+} | {
+    status: FlowRunStatus.FAILED
+    failedStep: FailedStep
+} | {
+    status: FlowRunStatus.RUNNING
+} 
+
+export class FlowExecutorContext {
+    tags: readonly string[]
+    steps: Readonly<Record<string, StepOutput>>
+    pauseRequestId: string
+    verdict: FlowVerdict
+    currentPath: StepExecutionPath
+    stepNameToTest?: boolean
+    stepsCount: number
+
+    /**
+     * Execution time in milliseconds
+     */
+    duration: number
+
+    constructor(copyFrom?: FlowExecutorContext) {
+        this.tags = copyFrom?.tags ?? []
+        this.steps = copyFrom?.steps ?? {}
+        this.pauseRequestId = copyFrom?.pauseRequestId ?? nanoid()
+        this.duration = copyFrom?.duration ?? -1
+        this.verdict = copyFrom?.verdict ?? { status: FlowRunStatus.RUNNING }
+        this.currentPath = copyFrom?.currentPath ?? StepExecutionPath.empty()
+        this.stepNameToTest = copyFrom?.stepNameToTest ?? false
+        this.stepsCount = copyFrom?.stepsCount ?? 0
+    }
+
+    static empty(): FlowExecutorContext {
+        return new FlowExecutorContext()
+    }
+
+    public setPauseRequestId(pauseRequestId: string): FlowExecutorContext {
+        return new FlowExecutorContext({
+            ...this,
+            pauseRequestId,
+        })
+    }
+
+    public getDelayedInSeconds(): number | undefined {
+        if (this.verdict.status === FlowRunStatus.PAUSED && this.verdict.pauseMetadata.type === PauseType.DELAY) {
+            return dayjs(this.verdict.pauseMetadata.resumeDateTime).diff(Date.now(), 'seconds')
+        }
+        return undefined
+    }
+
+    public finishExecution(): FlowExecutorContext {
+        if (this.verdict.status === FlowRunStatus.RUNNING) {
+            return new FlowExecutorContext({
+                ...this,
+                verdict: { status: FlowRunStatus.SUCCEEDED },
+            })
+        }
+        return this
+    }
+
+    public trimmedSteps(): Promise<Record<string, StepOutput>> {
+        return loggingUtils.trimExecution(this.steps)
+    }
+
+
+    public getLoopStepOutput({ stepName }: { stepName: string }): LoopStepOutput | undefined {
+        const stateAtPath = getStateAtPath({ currentPath: this.currentPath, steps: this.steps })
+        const stepOutput = stateAtPath[stepName]
+        if (isNil(stepOutput)) {
+            return undefined
+        }
+        assertEqual(stepOutput.type, FlowActionType.LOOP_ON_ITEMS, 'stepOutput.type', 'LOOP_ON_ITEMS')
+        // The new LoopStepOutput is needed as casting directly to LoopClassOutput will just cast the data but the class methods will not be available
+        return new LoopStepOutput(stepOutput as GenericStepOutput<FlowActionType.LOOP_ON_ITEMS, LoopStepResult>)
+    }
+
+    public isCompleted({ stepName }: { stepName: string }): boolean {
+        const stateAtPath = getStateAtPath({ currentPath: this.currentPath, steps: this.steps })
+        const stepOutput = stateAtPath[stepName]
+        if (isNil(stepOutput)) {
+            return false
+        }
+        return stepOutput.status !== StepOutputStatus.PAUSED
+    }
+
+    public isPaused({ stepName }: { stepName: string }): boolean {
+        const stateAtPath = getStateAtPath({ currentPath: this.currentPath, steps: this.steps })
+        const stepOutput = stateAtPath[stepName]
+        if (isNil(stepOutput)) {
+            return false
+        }
+        return stepOutput.status === StepOutputStatus.PAUSED
+    }
+
+    public setDuration(duration: number): FlowExecutorContext {
+        return new FlowExecutorContext({
+            ...this,
+            duration,
+        })
+    }
+
+
+    public addTags(tags: string[]): FlowExecutorContext {
+        return new FlowExecutorContext({
+            ...this,
+            tags: [...this.tags, ...tags].filter((value, index, self) => {
+                return self.indexOf(value) === index
+            }),
+        })
+    }
+
+    public upsertStep(stepName: string, stepOutput: StepOutput): FlowExecutorContext {
+        const steps = {
+            ...this.steps,
+        }
+        const targetMap = getStateAtPath({ currentPath: this.currentPath, steps })
+        targetMap[stepName] = stepOutput
+
+        return new FlowExecutorContext({
+            ...this,
+            steps,
+        })
+    }
+
+    public getStepOutput(stepName: string): StepOutput | undefined {
+        const stateAtPath = getStateAtPath({ currentPath: this.currentPath, steps: this.steps })
+        return stateAtPath[stepName]
+    }
+
+
+
+    public setCurrentPath(currentStatePath: StepExecutionPath): FlowExecutorContext {
+        return new FlowExecutorContext({
+            ...this,
+            currentPath: currentStatePath,
+        })
+    }
+
+    public setVerdict(verdict: FlowVerdict): FlowExecutorContext {
+        return new FlowExecutorContext({
+            ...this,
+            verdict,
+        })
+    }
+
+    public setRetryable(retryable: boolean): FlowExecutorContext {
+        return new FlowExecutorContext({
+            ...this,
+            retryable,
+        })
+    }
+
+    public incrementStepsExecuted(): FlowExecutorContext {
+        return new FlowExecutorContext({
+            ...this,
+            stepsCount: this.stepsCount + 1,
+        })
+    }
+
+   
+    public currentState(): Record<string, unknown> {
+        let flattenedSteps: Record<string, unknown> = extractOutput(this.steps)
+        let targetMap = this.steps
+        this.currentPath.path.forEach(([stepName, iteration]) => {
+            const stepOutput = targetMap[stepName]
+            if (!stepOutput.output || stepOutput.type !== FlowActionType.LOOP_ON_ITEMS) {
+                throw new EngineGenericError('NotInstanceOfLoopOnItemsStepOutputError', '[ExecutionState#getTargetMap] Not instance of Loop On Items step output')
+            }
+            targetMap = stepOutput.output.iterations[iteration]
+            flattenedSteps = {
+                ...flattenedSteps,
+                ...extractOutput(targetMap),
+            }
+        })
+        return flattenedSteps
+    }
+
+
+}
+
+function extractOutput(steps: Record<string, StepOutput>): Record<string, unknown> {
+    return Object.entries(steps).reduce((acc: Record<string, unknown>, [stepName, step]) => {
+        acc[stepName] = step.output
+        return acc
+    }, {} as Record<string, unknown>)
+}
+
+function getStateAtPath({ currentPath, steps }: { currentPath: StepExecutionPath, steps: Record<string, StepOutput> }): Record<string, StepOutput> {
+    let targetMap = steps
+    currentPath.path.forEach(([stepName, iteration]) => {
+        const stepOutput = targetMap[stepName]
+        if (!stepOutput.output || stepOutput.type !== FlowActionType.LOOP_ON_ITEMS) {
+            throw new EngineGenericError('NotInstanceOfLoopOnItemsStepOutputError', `[ExecutionState#getTargetMap] Not instance of Loop On Items step output: ${stepOutput.type}`)
+        }
+        targetMap = stepOutput.output.iterations[iteration]
+    })
+    return targetMap
+}
+
+

activepieces-fork/packages/engine/src/lib/handler/context/step-execution-path.ts

@@ -0,0 +1,20 @@
+export class StepExecutionPath {
+    public path: readonly [string, number][] = []
+
+    constructor(path: readonly [string, number][]) {
+        this.path = [...path]
+    }
+
+    loopIteration({ loopName, iteration }: { loopName: string, iteration: number }): StepExecutionPath {
+        return new StepExecutionPath([...this.path, [loopName, iteration]])
+    }
+
+    static empty(): StepExecutionPath {
+        return new StepExecutionPath([])
+    }
+
+    removeLast(): StepExecutionPath {
+        const newPath = this.path.slice(0, -1)
+        return new StepExecutionPath(newPath)
+    }
+}

activepieces-fork/packages/engine/src/lib/handler/context/test-execution-context.ts

@@ -0,0 +1,99 @@
+import { LATEST_CONTEXT_VERSION } from '@activepieces/pieces-framework'
+import {
+    FlowActionType,
+    flowStructureUtil,
+    FlowTriggerType,
+    FlowVersion,
+    GenericStepOutput,
+    isNil,
+    LoopStepOutput,
+    RouterStepOutput,
+    spreadIfDefined,
+    StepOutputStatus,
+} from '@activepieces/shared'
+import { createPropsResolver } from '../../variables/props-resolver'
+import { FlowExecutorContext } from './flow-execution-context'
+
+export const testExecutionContext = {
+    async stateFromFlowVersion({
+        flowVersion,
+        excludedStepName,
+        projectId,
+        engineToken,
+        apiUrl,
+        sampleData,
+    }: TestExecutionParams): Promise<FlowExecutorContext> {
+        let flowExecutionContext = FlowExecutorContext.empty()
+        if (isNil(flowVersion)) {
+            return flowExecutionContext
+        }
+        
+        const flowSteps = flowStructureUtil.getAllSteps(flowVersion.trigger)
+
+        for (const step of flowSteps) {
+            const { name } = step
+            if (name === excludedStepName) {
+                continue
+            }
+
+            const stepType = step.type
+            switch (stepType) {
+                case FlowActionType.ROUTER:
+                    flowExecutionContext = flowExecutionContext.upsertStep(
+                        step.name,
+                        RouterStepOutput.create({
+                            input: step.settings,
+                            type: stepType,
+                            status: StepOutputStatus.SUCCEEDED,
+                            ...spreadIfDefined('output', sampleData?.[step.name]),
+                        }),
+                    )
+                    break
+                case FlowActionType.LOOP_ON_ITEMS: {
+                    const { resolvedInput } = await createPropsResolver({
+                        apiUrl,
+                        projectId,
+                        engineToken,
+                        contextVersion: LATEST_CONTEXT_VERSION,
+                    }).resolve<{ items: unknown[] }>({
+                        unresolvedInput: step.settings,
+                        executionState: flowExecutionContext,
+                    })
+                    flowExecutionContext = flowExecutionContext.upsertStep(
+                        step.name,
+                        LoopStepOutput.init({
+                            input: step.settings,
+                        }).setOutput({
+                            item: resolvedInput.items[0],
+                            index: 1,
+                            iterations: [],
+                        }),
+                    )
+                    break
+                }
+                case FlowActionType.PIECE:
+                case FlowActionType.CODE:
+                case FlowTriggerType.EMPTY:
+                case FlowTriggerType.PIECE:
+                    flowExecutionContext = flowExecutionContext.upsertStep(step.name, GenericStepOutput.create({
+                        input: {},
+                        type: stepType,
+                        status: StepOutputStatus.SUCCEEDED,
+                        ...spreadIfDefined('output', sampleData?.[step.name]),
+                    }))
+                    break
+            }
+        }
+        return flowExecutionContext
+    },
+}
+
+
+type TestExecutionParams = {
+    flowVersion?: FlowVersion
+    excludedStepName?: string
+    projectId: string
+    apiUrl: string
+    engineToken: string
+    sampleData?: Record<string, unknown>
+}

activepieces-fork/packages/engine/src/lib/handler/flow-executor.ts

@@ -0,0 +1,90 @@
+import { performance } from 'node:perf_hooks'
+import { EngineGenericError, ExecuteFlowOperation, ExecutionType, FlowAction, FlowActionType, FlowRunStatus, isNil } from '@activepieces/shared'
+import { triggerHelper } from '../helper/trigger-helper'
+import { progressService } from '../services/progress.service'
+import { BaseExecutor } from './base-executor'
+import { codeExecutor } from './code-executor'
+import { EngineConstants } from './context/engine-constants'
+import { FlowExecutorContext } from './context/flow-execution-context'
+import { loopExecutor } from './loop-executor'
+import { pieceExecutor } from './piece-executor'
+import { routerExecuter } from './router-executor'
+
+function getExecuteFunction(): Record<FlowActionType, BaseExecutor<FlowAction>> {
+    return {
+        [FlowActionType.CODE]: codeExecutor,
+        [FlowActionType.LOOP_ON_ITEMS]: loopExecutor,
+        [FlowActionType.PIECE]: pieceExecutor,
+        [FlowActionType.ROUTER]: routerExecuter,
+    }
+}
+
+export const flowExecutor = {
+    getExecutorForAction(type: FlowActionType): BaseExecutor<FlowAction> {
+        const executeFunction = getExecuteFunction()
+        const executor = executeFunction[type]
+
+        if (isNil(executor)) {
+            throw new EngineGenericError('ExecutorNotFoundError', `Executor not found for action type: ${type}`)
+        }
+        
+        return executor
+    },
+    async executeFromTrigger({ executionState, constants, input }: {
+        executionState: FlowExecutorContext
+        constants: EngineConstants
+        input: ExecuteFlowOperation
+    }): Promise<FlowExecutorContext> {
+        const trigger = input.flowVersion.trigger
+        if (input.executionType === ExecutionType.BEGIN) {
+            await triggerHelper.executeOnStart(trigger, constants, input.triggerPayload)
+        }
+        return flowExecutor.execute({
+            action: trigger.nextAction,
+            executionState,
+            constants,
+        })
+    },
+    async execute({ action, constants, executionState }: {
+        action: FlowAction | null | undefined
+        executionState: FlowExecutorContext
+        constants: EngineConstants
+    }): Promise<FlowExecutorContext> {
+        const flowStartTime = performance.now()
+        let flowExecutionContext = executionState
+        let currentAction: FlowAction | null | undefined = action
+
+        while (!isNil(currentAction)) {
+            const testSingleStepMode = !isNil(constants.stepNameToTest)
+            if (currentAction.skip && !testSingleStepMode) {
+                currentAction = currentAction.nextAction
+                continue
+            }
+            const handler = this.getExecutorForAction(currentAction.type)
+
+            progressService.sendUpdate({
+                engineConstants: constants,
+                flowExecutorContext: flowExecutionContext,
+            }).catch(error => {
+                console.error('Error sending update:', error)
+            })
+
+            flowExecutionContext = await handler.handle({
+                action: currentAction,
+                executionState: flowExecutionContext,
+                constants,
+            })
+            const shouldBreakExecution = flowExecutionContext.verdict.status !== FlowRunStatus.RUNNING || testSingleStepMode
+
+            if (shouldBreakExecution) {
+                break
+            }
+
+            currentAction = currentAction.nextAction
+        }
+
+        const flowEndTime = performance.now()
+        return flowExecutionContext.setDuration(flowEndTime - flowStartTime)
+    },
+}
+

activepieces-fork/packages/engine/src/lib/handler/loop-executor.ts

@@ -0,0 +1,77 @@
+import { LATEST_CONTEXT_VERSION } from '@activepieces/pieces-framework'
+import { FlowRunStatus, isNil, LoopOnItemsAction, LoopStepOutput, StepOutputStatus } from '@activepieces/shared'
+import { BaseExecutor } from './base-executor'
+import { flowExecutor } from './flow-executor'
+
+type LoopOnActionResolvedSettings = {
+    items: readonly unknown[]
+}
+
+export const loopExecutor: BaseExecutor<LoopOnItemsAction> = {
+    async handle({
+        action,
+        executionState,
+        constants,
+    }) {
+        const stepStartTime = performance.now()
+        const { resolvedInput, censoredInput } = await constants.getPropsResolver(LATEST_CONTEXT_VERSION).resolve<LoopOnActionResolvedSettings>({
+            unresolvedInput: {
+                items: action.settings.items,
+            },
+            executionState,
+        })
+        const previousStepOutput = executionState.getLoopStepOutput({ stepName: action.name })
+        let stepOutput = previousStepOutput ?? LoopStepOutput.init({
+            input: censoredInput,
+        })
+        let newExecutionContext = executionState.upsertStep(action.name, stepOutput)
+
+        if (!Array.isArray(resolvedInput.items)) {
+            const errorMessage = JSON.stringify({
+                message: 'The items you have selected must be a list.',
+            })
+            const failedStepOutput = stepOutput
+                .setStatus(StepOutputStatus.FAILED)
+                .setErrorMessage(errorMessage)
+                .setDuration( performance.now() - stepStartTime)
+            return newExecutionContext.upsertStep(action.name, failedStepOutput).setVerdict({ status: FlowRunStatus.FAILED, failedStep: {
+                name: action.name,
+                displayName: action.displayName,
+                message: errorMessage,
+            } })
+        }
+
+        const firstLoopAction = action.firstLoopAction
+
+
+        for (let i = 0; i < resolvedInput.items.length; ++i) {
+            const newCurrentPath = newExecutionContext.currentPath.loopIteration({ loopName: action.name, iteration: i })
+
+            const testSingleStepMode = !isNil(constants.stepNameToTest)
+            stepOutput = stepOutput.setItemAndIndex({ item: resolvedInput.items[i], index: i + 1 })
+            const addEmptyIteration = !stepOutput.hasIteration(i)
+            if (addEmptyIteration) {
+                stepOutput = stepOutput.addIteration()
+            }
+            newExecutionContext = newExecutionContext.upsertStep(action.name, stepOutput).setCurrentPath(newCurrentPath)
+            if (!isNil(firstLoopAction) && !testSingleStepMode) {
+                newExecutionContext = await flowExecutor.execute({
+                    action: firstLoopAction,
+                    executionState: newExecutionContext,
+                    constants,
+                })
+            }
+
+            newExecutionContext = newExecutionContext.setCurrentPath(newExecutionContext.currentPath.removeLast())
+
+            if (newExecutionContext.verdict.status !== FlowRunStatus.RUNNING) {
+                return newExecutionContext.upsertStep(action.name, stepOutput.setDuration(performance.now() - stepStartTime))
+            }
+
+            if (testSingleStepMode) {
+                break
+            }
+        }
+        return newExecutionContext.upsertStep(action.name, stepOutput.setDuration(performance.now() - stepStartTime))
+    },
+}

activepieces-fork/packages/engine/src/lib/handler/piece-executor.ts

@@ -0,0 +1,325 @@
+import { URL } from 'url'
+import { ActionContext, backwardCompatabilityContextUtils, ConstructToolParams, InputPropertyMap, PauseHook, PauseHookParams, PieceAuthProperty, PiecePropertyMap, RespondHook, RespondHookParams, StaticPropsValue, StopHook, StopHookParams, TagsManager } from '@activepieces/pieces-framework'
+import { AUTHENTICATION_PROPERTY_NAME, EngineGenericError, EngineSocketEvent, ExecutionType, FlowActionType, FlowRunStatus, GenericStepOutput, isNil, PausedFlowTimeoutError, PauseType, PieceAction, RespondResponse, StepOutputStatus } from '@activepieces/shared'
+import { LanguageModelV2 } from '@ai-sdk/provider'
+import { ToolSet } from 'ai'
+import dayjs from 'dayjs'
+import { continueIfFailureHandler, runWithExponentialBackoff } from '../helper/error-handling'
+import { pieceLoader } from '../helper/piece-loader'
+import { createFlowsContext } from '../services/flows.service'
+import { progressService } from '../services/progress.service'
+import { createFilesService } from '../services/step-files.service'
+import { createContextStore } from '../services/storage.service'
+import { agentTools } from '../tools'
+import { HookResponse, utils } from '../utils'
+import { propsProcessor } from '../variables/props-processor'
+import { workerSocket } from '../worker-socket'
+import { ActionHandler, BaseExecutor } from './base-executor'
+
+const AP_PAUSED_FLOW_TIMEOUT_DAYS = Number(process.env.AP_PAUSED_FLOW_TIMEOUT_DAYS)
+
+export const pieceExecutor: BaseExecutor<PieceAction> = {
+    async handle({
+        action,
+        executionState,
+        constants,
+    }) {
+        if (executionState.isCompleted({ stepName: action.name })) {
+            return executionState
+        }
+        const resultExecution = await runWithExponentialBackoff(executionState, action, constants, executeAction)
+        return continueIfFailureHandler(resultExecution, action, constants)
+    },
+}
+
+const executeAction: ActionHandler<PieceAction> = async ({ action, executionState, constants }) => {
+    const stepStartTime = performance.now()
+    const stepOutput = GenericStepOutput.create({
+        input: {},
+        type: FlowActionType.PIECE,
+        status: StepOutputStatus.RUNNING,
+    })
+
+    const { data: executionStateResult, error: executionStateError } = await utils.tryCatchAndThrowOnEngineError((async () => {
+        if (isNil(action.settings.actionName)) {
+            throw new EngineGenericError('ActionNameNotSetError', 'Action name is not set')
+        }
+        
+        const { pieceAction, piece } = await pieceLoader.getPieceAndActionOrThrow({
+            pieceName: action.settings.pieceName,
+            pieceVersion: action.settings.pieceVersion,
+            actionName: action.settings.actionName,
+            devPieces: constants.devPieces,
+        })
+
+        const { resolvedInput, censoredInput } = await constants.getPropsResolver(piece.getContextInfo?.().version).resolve<StaticPropsValue<PiecePropertyMap>>({
+            unresolvedInput: action.settings.input,
+            executionState,
+        })
+
+        stepOutput.input = censoredInput
+    
+        const { processedInput, errors } = await propsProcessor.applyProcessorsAndValidators(resolvedInput, pieceAction.props, piece.auth, pieceAction.requireAuth, action.settings.propertySettings)
+        if (Object.keys(errors).length > 0) {
+            throw new Error(JSON.stringify(errors, null, 2))
+        }
+
+
+        const params: {
+            hookResponse: HookResponse
+        } = {
+            hookResponse: {
+                type: 'none',
+                tags: [],
+            },
+        }
+        const outputContext = progressService.createOutputContext({
+            engineConstants: constants,
+            flowExecutorContext: executionState,
+            stepName: action.name,
+            stepOutput,
+        })
+
+        const isPaused = executionState.isPaused({ stepName: action.name })
+        if (!isPaused) {
+            await progressService.sendUpdate({
+                engineConstants: constants,
+                flowExecutorContext: executionState.upsertStep(action.name, stepOutput),
+            })
+        }
+        const context: ActionContext<PieceAuthProperty, InputPropertyMap> = {
+            executionType: isPaused ? ExecutionType.RESUME : ExecutionType.BEGIN,
+            resumePayload: constants.resumePayload!,
+            store: createContextStore({
+                apiUrl: constants.internalApiUrl,
+                prefix: '',
+                flowId: constants.flowId,
+                engineToken: constants.engineToken,
+            }),
+            output: outputContext,
+            flows: createFlowsContext({
+                engineToken: constants.engineToken,
+                internalApiUrl: constants.internalApiUrl,
+                flowId: constants.flowId,
+                flowVersionId: constants.flowVersionId,
+            }),
+            step: {
+                name: action.name,
+            },
+            auth: processedInput[AUTHENTICATION_PROPERTY_NAME],
+            files: createFilesService({
+                apiUrl: constants.internalApiUrl,
+                engineToken: constants.engineToken,
+                stepName: action.name,
+                flowId: constants.flowId,
+            }),
+            server: {
+                token: constants.engineToken,
+                apiUrl: constants.internalApiUrl,
+                publicUrl: constants.publicApiUrl,
+            },
+            agent: {
+                tools: async (params: ConstructToolParams): Promise<ToolSet> => agentTools.tools({
+                    engineConstants: constants,
+                    tools: params.tools,
+                    model: params.model as LanguageModelV2,
+                }),
+            },
+            propsValue: processedInput,
+            tags: createTagsManager(params),
+            connections: utils.createConnectionManager({
+                apiUrl: constants.internalApiUrl,
+                projectId: constants.projectId,
+                engineToken: constants.engineToken,
+                target: 'actions',
+                hookResponse: params.hookResponse,
+                contextVersion: piece.getContextInfo?.().version,
+            }),
+            run: {
+                id: constants.flowRunId,
+                stop: createStopHook(params),
+                pause: createPauseHook(params, executionState.pauseRequestId, constants.httpRequestId),
+                respond: createRespondHook(params),
+            },
+            project: {
+                id: constants.projectId,
+                externalId: constants.externalProjectId,
+            },
+            generateResumeUrl: (params) => {
+                const url = new URL(`${constants.publicApiUrl}v1/flow-runs/${constants.flowRunId}/requests/${executionState.pauseRequestId}${params.sync ? '/sync' : ''}`)
+                url.search = new URLSearchParams(params.queryParams).toString()
+                return url.toString()
+            },
+        }
+        const backwardCompatibleContext = backwardCompatabilityContextUtils.makeActionContextBackwardCompatible({
+            contextVersion: piece.getContextInfo?.().version,
+            context,
+        })
+        const testSingleStepMode = !isNil(constants.stepNameToTest)
+        const runMethodToExecute = (testSingleStepMode && !isNil(pieceAction.test)) ? pieceAction.test : pieceAction.run
+        const output = await runMethodToExecute(backwardCompatibleContext)
+        const newExecutionContext = executionState.addTags(params.hookResponse.tags)
+
+        const webhookResponse = getResponse(params.hookResponse)
+        const isSamePiece = constants.triggerPieceName === action.settings.pieceName
+        if (!isNil(webhookResponse) && !isNil(constants.serverHandlerId) && !isNil(constants.httpRequestId) && isSamePiece) {
+            await workerSocket.sendToWorkerWithAck(EngineSocketEvent.SEND_FLOW_RESPONSE, {
+                workerHandlerId: constants.serverHandlerId,
+                httpRequestId: constants.httpRequestId,
+                runResponse: {
+                    status: webhookResponse.status ?? 200,
+                    body: webhookResponse.body ?? {},
+                    headers: webhookResponse.headers ?? {},
+                },
+            })
+        }
+
+        const stepEndTime = performance.now()
+        if (params.hookResponse.type === 'stopped') {
+            if (isNil(params.hookResponse.response)) {
+                throw new EngineGenericError('StopResponseNotSetError', 'Stop response is not set')
+            }
+
+            return newExecutionContext.upsertStep(action.name, stepOutput.setOutput(output).setStatus(StepOutputStatus.SUCCEEDED).setDuration(stepEndTime - stepStartTime)).incrementStepsExecuted().setVerdict({
+                status: FlowRunStatus.SUCCEEDED,
+                stopResponse: (params.hookResponse.response as StopHookParams).response,
+            })
+        }
+        if (params.hookResponse.type === 'paused') {
+            if (isNil(params.hookResponse.response)) {
+                throw new EngineGenericError('PauseResponseNotSetError', 'Pause response is not set')
+            }
+
+            return newExecutionContext.upsertStep(action.name, stepOutput.setOutput(output).setStatus(StepOutputStatus.PAUSED).setDuration(stepEndTime - stepStartTime)).incrementStepsExecuted()
+                .setVerdict({
+                    status: FlowRunStatus.PAUSED,
+                    pauseMetadata: (params.hookResponse.response as PauseHookParams).pauseMetadata,
+                })
+        }
+        return newExecutionContext.upsertStep(action.name, stepOutput.setOutput(output).setStatus(StepOutputStatus.SUCCEEDED).setDuration(stepEndTime - stepStartTime)).incrementStepsExecuted().setVerdict({ status: FlowRunStatus.RUNNING })
+
+    }))
+
+    if (executionStateError) {
+        const failedStepOutput = stepOutput
+            .setStatus(StepOutputStatus.FAILED)
+            .setErrorMessage(utils.formatError(executionStateError))
+            .setDuration(performance.now() - stepStartTime)
+
+        return executionState
+            .upsertStep(action.name, failedStepOutput)
+            .setVerdict({
+                status: FlowRunStatus.FAILED, failedStep: {
+                    name: action.name,
+                    displayName: action.displayName,
+                    message: utils.formatError(executionStateError),
+                },
+            })
+    }
+
+    return executionStateResult
+}
+
+function getResponse(hookResponse: HookResponse): RespondResponse | undefined {
+    switch (hookResponse.type) {
+        case 'stopped':
+        case 'respond':
+            return hookResponse.response.response
+        case 'paused':
+            if (hookResponse.response.pauseMetadata.type === PauseType.WEBHOOK) {
+                return hookResponse.response.pauseMetadata.response
+            }
+            else {
+                return undefined
+            }
+        case 'none':
+            return undefined
+    }
+}
+
+const createTagsManager = (hkParams: createTagsManagerParams): TagsManager => {
+    return {
+        add: async (params: addTagsParams): Promise<void> => {
+            hkParams.hookResponse.tags.push(params.name)
+        },
+
+    }
+}
+
+type addTagsParams = {
+    name: string
+}
+
+type createTagsManagerParams = {
+    hookResponse: HookResponse
+}
+
+
+function createStopHook(params: CreateStopHookParams): StopHook {
+    return (req?: StopHookParams) => {
+        params.hookResponse = {
+            ...params.hookResponse,
+            type: 'stopped',
+            response: req ?? { response: {} },
+        }
+    }
+}
+type CreateStopHookParams = {
+    hookResponse: HookResponse
+}
+
+function createRespondHook(params: CreateRespondHookParams): RespondHook {
+    return (req?: RespondHookParams) => {
+        params.hookResponse = {
+            ...params.hookResponse,
+            type: 'respond',
+            response: req ?? { response: {} },
+        }
+    }
+}
+
+type CreateRespondHookParams = {
+    hookResponse: HookResponse
+}
+
+function createPauseHook(params: CreatePauseHookParams, pauseId: string, requestIdToReply: string | null): PauseHook {
+    return (req) => {
+        switch (req.pauseMetadata.type) {
+            case PauseType.DELAY: {
+                const diffInDays = dayjs(req.pauseMetadata.resumeDateTime).diff(dayjs(), 'days')
+                if (diffInDays > AP_PAUSED_FLOW_TIMEOUT_DAYS) {
+                    throw new PausedFlowTimeoutError(undefined, AP_PAUSED_FLOW_TIMEOUT_DAYS)
+                }
+                params.hookResponse = {
+                    ...params.hookResponse,
+                    type: 'paused',
+                    response: {
+                        pauseMetadata: {
+                            ...req.pauseMetadata,
+                            requestIdToReply: requestIdToReply ?? undefined,
+                        },
+                    },
+                }
+                break
+            }
+            case PauseType.WEBHOOK:
+                params.hookResponse = {
+                    ...params.hookResponse,
+                    type: 'paused',
+                    response: {
+                        pauseMetadata: {
+                            ...req.pauseMetadata,
+                            requestId: pauseId,
+                            requestIdToReply: requestIdToReply ?? undefined,
+                            response: req.pauseMetadata.response ?? {},
+                        },
+                    },
+                }
+                break
+        }
+    }
+}
+
+type CreatePauseHookParams = {
+    hookResponse: HookResponse
+}

activepieces-fork/packages/engine/src/lib/handler/router-executor.ts

@@ -0,0 +1,284 @@
+import { LATEST_CONTEXT_VERSION } from '@activepieces/pieces-framework'
+import { BranchCondition, BranchExecutionType, BranchOperator, EngineGenericError, FlowRunStatus, isNil, RouterAction, RouterActionSettings, RouterExecutionType, RouterStepOutput, StepOutputStatus } from '@activepieces/shared'
+import dayjs from 'dayjs'
+import { utils } from '../utils'
+import { BaseExecutor } from './base-executor'
+import { EngineConstants } from './context/engine-constants'
+import { FlowExecutorContext } from './context/flow-execution-context'
+import { flowExecutor } from './flow-executor'
+
+export const routerExecuter: BaseExecutor<RouterAction> = {
+    async handle({
+        action,
+        executionState,
+        constants,
+    }) {
+        const { censoredInput, resolvedInput } = await constants.getPropsResolver(LATEST_CONTEXT_VERSION).resolve<RouterActionSettings>({
+            unresolvedInput: {
+                ...action.settings,
+            },
+            executionState,
+        })
+
+        switch (resolvedInput.executionType) {
+            case RouterExecutionType.EXECUTE_ALL_MATCH:
+                return handleRouterExecution({ action, executionState, constants, censoredInput, resolvedInput, routerExecutionType: RouterExecutionType.EXECUTE_ALL_MATCH })
+            case RouterExecutionType.EXECUTE_FIRST_MATCH:
+                return handleRouterExecution({ action, executionState, constants, censoredInput, resolvedInput, routerExecutionType: RouterExecutionType.EXECUTE_FIRST_MATCH })
+            default:
+                throw new EngineGenericError('RouterExecutionTypeNotSupportedError', `Router execution type ${resolvedInput.executionType} is not supported`)
+        }
+    },
+}
+
+async function handleRouterExecution({ action, executionState, constants, censoredInput, resolvedInput, routerExecutionType }: {
+    action: RouterAction
+    executionState: FlowExecutorContext
+    constants: EngineConstants
+    censoredInput: unknown
+    resolvedInput: RouterActionSettings
+    routerExecutionType: RouterExecutionType
+}): Promise<FlowExecutorContext> {
+    const stepStartTime = performance.now()
+
+    const evaluatedConditionsWithoutFallback = resolvedInput.branches.map((branch) => {
+        return branch.branchType === BranchExecutionType.FALLBACK ? true : evaluateConditions(branch.conditions)
+    })
+
+    const evaluatedConditions = resolvedInput.branches.map((branch, index) => {
+        if (branch.branchType === BranchExecutionType.CONDITION) {
+            return evaluatedConditionsWithoutFallback[index]
+        }
+        const fallback = evaluatedConditionsWithoutFallback.filter((_, i) => i !== index).every((condition) => !condition)
+        return fallback
+    })
+
+    const stepEndTime = performance.now()
+    const routerOutput = RouterStepOutput.init({
+        input: censoredInput,
+    }).setOutput({
+        branches: resolvedInput.branches.map((branch, index) => ({
+            branchName: branch.branchName,
+            branchIndex: index + 1,
+            evaluation: evaluatedConditions[index],
+        })),
+    }).setDuration(stepEndTime - stepStartTime)
+    executionState = executionState.upsertStep(action.name, routerOutput)
+
+    const { data: executionStateResult, error: executionStateError } = await utils.tryCatchAndThrowOnEngineError(async () => {
+        for (let i = 0; i < resolvedInput.branches.length; i++) {
+            if (!isNil(constants.stepNameToTest)) {
+                break
+            }
+            const condition = routerOutput.output?.branches[i].evaluation
+            if (!condition) {
+                continue
+            }
+    
+            executionState = await flowExecutor.execute({
+                action: action.children[i],
+                executionState,
+                constants,
+            })
+    
+            const shouldBreakExecution = executionState.verdict.status !== FlowRunStatus.RUNNING || routerExecutionType === RouterExecutionType.EXECUTE_FIRST_MATCH
+            if (shouldBreakExecution) {
+                break
+            }
+        }
+        return executionState
+    })
+    if (executionStateError) {
+        const failedStepOutput = routerOutput.setStatus(StepOutputStatus.FAILED)
+        return executionState.upsertStep(action.name, failedStepOutput).setVerdict({ status: FlowRunStatus.FAILED, failedStep: {
+            name: action.name,
+            displayName: action.displayName,
+            message: utils.formatError(executionStateError),
+        } })
+    }
+
+    return executionStateResult
+}
+
+
+export function evaluateConditions(conditionGroups: BranchCondition[][]): boolean {
+    let orOperator = false
+    for (const conditionGroup of conditionGroups) {
+        let andGroup = true
+        for (const condition of conditionGroup) {
+            const castedCondition = condition
+
+            if (isNil(castedCondition.operator)) {
+                throw new EngineGenericError('OperatorNotSetError', 'The operator is required but found to be undefined')
+            }
+
+            switch (castedCondition.operator) {
+                case BranchOperator.TEXT_CONTAINS: {
+                    const firstValueContains = toLowercaseIfCaseInsensitive(castedCondition.firstValue, castedCondition.caseSensitive).includes(
+                        toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive),
+                    )
+                    andGroup = andGroup && firstValueContains
+                    break
+                }
+                case BranchOperator.TEXT_DOES_NOT_CONTAIN: {
+                    const firstValueDoesNotContain = !toLowercaseIfCaseInsensitive(castedCondition.firstValue, castedCondition.caseSensitive).includes(
+                        toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive),
+                    )
+                    andGroup = andGroup && firstValueDoesNotContain
+                    break
+                }
+                case BranchOperator.TEXT_EXACTLY_MATCHES: {
+                    const firstValueExactlyMatches = toLowercaseIfCaseInsensitive(castedCondition.firstValue, castedCondition.caseSensitive) ===
+                        toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive)
+                    andGroup = andGroup && firstValueExactlyMatches
+                    break
+                }
+                case BranchOperator.TEXT_DOES_NOT_EXACTLY_MATCH: {
+                    const firstValueDoesNotExactlyMatch = toLowercaseIfCaseInsensitive(castedCondition.firstValue, castedCondition.caseSensitive) !==
+                        toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive)
+                    andGroup = andGroup && firstValueDoesNotExactlyMatch
+                    break
+                }
+                case BranchOperator.TEXT_STARTS_WITH: {
+                    const firstValueStartsWith = toLowercaseIfCaseInsensitive(castedCondition.firstValue, castedCondition.caseSensitive).startsWith(
+                        toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive),
+                    )
+                    andGroup = andGroup && firstValueStartsWith
+                    break
+                }
+                case BranchOperator.TEXT_ENDS_WITH: {
+                    const firstValueEndsWith = toLowercaseIfCaseInsensitive(castedCondition.firstValue, castedCondition.caseSensitive).endsWith(
+                        toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive),
+                    )
+                    andGroup = andGroup && firstValueEndsWith
+                    break
+                }
+                case BranchOperator.TEXT_DOES_NOT_START_WITH: {
+                    const firstValueDoesNotStartWith = !toLowercaseIfCaseInsensitive(castedCondition.firstValue, castedCondition.caseSensitive).startsWith(
+                        toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive),
+                    )
+                    andGroup = andGroup && firstValueDoesNotStartWith
+                    break
+                }
+                case BranchOperator.TEXT_DOES_NOT_END_WITH: {
+                    const firstValueDoesNotEndWith = !toLowercaseIfCaseInsensitive(castedCondition.firstValue, castedCondition.caseSensitive).endsWith(
+                        toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive),
+                    )
+                    andGroup = andGroup && firstValueDoesNotEndWith
+                    break
+                }
+                case BranchOperator.LIST_CONTAINS: {
+                    const list = parseAndCoerceListAsArray(castedCondition.firstValue)
+                    andGroup = andGroup && list.some((item) =>
+                        toLowercaseIfCaseInsensitive(item, castedCondition.caseSensitive) === toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive),
+                    )
+                    break
+                }
+                case BranchOperator.LIST_DOES_NOT_CONTAIN: {
+                    const list = parseAndCoerceListAsArray(castedCondition.firstValue)
+                    andGroup = andGroup && !list.some((item) =>
+                        toLowercaseIfCaseInsensitive(item, castedCondition.caseSensitive) === toLowercaseIfCaseInsensitive(castedCondition.secondValue, castedCondition.caseSensitive),
+                    )
+                    break
+                }
+                case BranchOperator.NUMBER_IS_GREATER_THAN: {
+                    const firstValue = parseStringToNumber(castedCondition.firstValue)
+                    const secondValue = parseStringToNumber(castedCondition.secondValue)
+                    andGroup = andGroup && firstValue > secondValue
+                    break
+                }
+                case BranchOperator.NUMBER_IS_LESS_THAN: {
+                    const firstValue = parseStringToNumber(castedCondition.firstValue)
+                    const secondValue = parseStringToNumber(castedCondition.secondValue)
+                    andGroup = andGroup && firstValue < secondValue
+                    break
+                }
+                case BranchOperator.NUMBER_IS_EQUAL_TO: {
+                    const firstValue = parseStringToNumber(castedCondition.firstValue)
+                    const secondValue = parseStringToNumber(castedCondition.secondValue)
+                    andGroup = andGroup && firstValue == secondValue
+                    break
+                }
+                case BranchOperator.BOOLEAN_IS_TRUE:
+                    andGroup = andGroup && !!castedCondition.firstValue
+                    break
+                case BranchOperator.BOOLEAN_IS_FALSE:
+                    andGroup = andGroup && !castedCondition.firstValue
+                    break
+                case BranchOperator.DATE_IS_AFTER:
+                    andGroup = andGroup && isValidDate(castedCondition.firstValue) && isValidDate(castedCondition.secondValue) && dayjs(castedCondition.firstValue).isAfter(dayjs(castedCondition.secondValue))
+                    break
+                case BranchOperator.DATE_IS_EQUAL:
+                    andGroup = andGroup && isValidDate(castedCondition.firstValue) && isValidDate(castedCondition.secondValue) && dayjs(castedCondition.firstValue).isSame(dayjs(castedCondition.secondValue))
+                    break
+                case BranchOperator.DATE_IS_BEFORE:
+                    andGroup = andGroup && isValidDate(castedCondition.firstValue) && isValidDate(castedCondition.secondValue) && dayjs(castedCondition.firstValue).isBefore(dayjs(castedCondition.secondValue))
+                    break
+                case BranchOperator.LIST_IS_EMPTY: {
+                    const list = parseListAsArray(castedCondition.firstValue)
+                    andGroup = andGroup && Array.isArray(list) && list?.length === 0
+                    break
+                }
+                case BranchOperator.LIST_IS_NOT_EMPTY: {
+                    const list = parseListAsArray(castedCondition.firstValue)
+                    andGroup = andGroup && Array.isArray(list) && list?.length !== 0
+                    break
+                }
+                case BranchOperator.EXISTS:
+                    andGroup = andGroup && castedCondition.firstValue !== undefined && castedCondition.firstValue !== null && castedCondition.firstValue !== ''
+                    break
+                case BranchOperator.DOES_NOT_EXIST:
+                    andGroup = andGroup && (castedCondition.firstValue === undefined || castedCondition.firstValue === null || castedCondition.firstValue === '')
+                    break
+            }
+        }
+        orOperator = orOperator || andGroup
+    }
+    return Boolean(orOperator)
+}
+
+function toLowercaseIfCaseInsensitive(text: unknown, caseSensitive: boolean | undefined): string {
+    if (typeof text === 'string') {
+        return caseSensitive ? text : text.toLowerCase()
+    }
+    const textAsString = JSON.stringify(text)
+    return caseSensitive ? textAsString : textAsString.toLowerCase()
+}
+
+function parseStringToNumber(str: string): number | string {
+    const num = Number(str)
+    return isNaN(num) ? str : num
+}
+
+function parseListAsArray(input: unknown): unknown[] | undefined {
+    if (typeof input === 'string') {
+        try {
+            const parsed = JSON.parse(input)
+            return Array.isArray(parsed) ? parsed : undefined
+        }
+        catch (e) {
+            return undefined
+        }
+    }
+    return Array.isArray(input) ? input : undefined
+}
+
+function parseAndCoerceListAsArray(input: unknown): unknown[] {
+    if (typeof input === 'string') {
+        try {
+            const parsed = JSON.parse(input)
+            return Array.isArray(parsed) ? parsed : [parsed]
+        }
+        catch (e) {
+            return [input]
+        }
+    }
+    return Array.isArray(input) ? input : [input]
+}
+
+function isValidDate(date: unknown): boolean {
+    if (typeof date === 'string' || typeof date === 'number' || date instanceof Date) {
+        return dayjs(date).isValid()
+    }
+    return false
+}

activepieces-fork/packages/engine/src/lib/helper/error-handling.ts

@@ -0,0 +1,59 @@
+import { CodeAction, FlowRunStatus, isNil, PieceAction } from '@activepieces/shared'
+import { EngineConstants } from '../handler/context/engine-constants'
+import {  FlowExecutorContext } from '../handler/context/flow-execution-context'
+
+export async function runWithExponentialBackoff<T extends CodeAction | PieceAction>(
+    executionState: FlowExecutorContext,
+    action: T,
+    constants: EngineConstants,
+    requestFunction: RequestFunction<T>,
+    attemptCount = 1,
+): Promise<FlowExecutorContext> {
+    const resultExecutionState = await requestFunction({ action, executionState, constants })
+    const retryEnabled = action.settings.errorHandlingOptions?.retryOnFailure?.value
+    if (
+        executionFailedWithRetryableError(resultExecutionState) &&
+        attemptCount < constants.retryConstants.maxAttempts &&
+        retryEnabled &&
+        isNil(constants.stepNameToTest)
+    ) {
+        const backoffTime = Math.pow(constants.retryConstants.retryExponential, attemptCount) * constants.retryConstants.retryInterval
+        await new Promise(resolve => setTimeout(resolve, backoffTime))
+        return runWithExponentialBackoff(executionState, action, constants, requestFunction, attemptCount + 1)
+    }
+
+    return resultExecutionState
+}
+
+export async function continueIfFailureHandler(
+    executionState: FlowExecutorContext,
+    action: CodeAction | PieceAction,
+    constants: EngineConstants,
+): Promise<FlowExecutorContext> {
+    const continueOnFailure = action.settings.errorHandlingOptions?.continueOnFailure?.value
+
+    if (
+        executionState.verdict.status === FlowRunStatus.FAILED &&
+        continueOnFailure &&
+        isNil(constants.stepNameToTest)
+    ) {
+        return executionState
+            .setVerdict({ status: FlowRunStatus.RUNNING })
+    }
+
+    return executionState
+}
+
+
+const executionFailedWithRetryableError = (flowExecutorContext: FlowExecutorContext): boolean => {
+    return flowExecutorContext.verdict.status === FlowRunStatus.FAILED
+}
+
+type Request<T extends CodeAction | PieceAction> = {
+    action: T
+    executionState: FlowExecutorContext
+    constants: EngineConstants
+}
+
+type RequestFunction<T extends CodeAction | PieceAction> = (request: Request<T>) => Promise<FlowExecutorContext>
+

activepieces-fork/packages/engine/src/lib/helper/logging-utils.ts

@@ -0,0 +1,128 @@
+import { isObject, StepOutput } from '@activepieces/shared'
+import { Queue } from '@datastructures-js/queue'
+import sizeof from 'object-sizeof'
+import PriorityQueue from 'priority-queue-typescript'
+
+const TRUNCATION_TEXT_PLACEHOLDER = '(truncated)'
+const ERROR_OFFSET = 256 * 1024
+const DEFAULT_MAX_LOG_SIZE_FOR_TESTING = '10'
+const MAX_LOG_SIZE = Number(process.env.AP_MAX_FILE_SIZE_MB ?? DEFAULT_MAX_LOG_SIZE_FOR_TESTING) * 1024 * 1024
+const MAX_SIZE_FOR_ALL_ENTRIES = MAX_LOG_SIZE - ERROR_OFFSET
+const SIZE_OF_TRUNCATION_TEXT_PLACEHOLDER = sizeof(TRUNCATION_TEXT_PLACEHOLDER)
+const nonTruncatableKeys: Key[] = ['status', 'duration', 'type']
+
+export const loggingUtils = {
+    async trimExecution(steps: Record<string, StepOutput>): Promise<Record<string, StepOutput>> {
+        const totalJsonSize = sizeof(steps)
+        if (!jsonExceedMaxSize(totalJsonSize)) {
+            return steps
+        }
+        return removeLeavesInTopologicalOrder(JSON.parse(JSON.stringify(steps)))
+    },
+}
+
+function removeLeavesInTopologicalOrder(json: Record<string, unknown>): Record<string, StepOutput> {
+    const nodes: Node[] = traverseJsonAndConvertToNodes(json)
+    const leaves = new PriorityQueue<Node>(
+        undefined,
+        (a: Node, b: Node) => b.size - a.size,
+    )
+    nodes.filter((node) => node.numberOfChildren === 0).forEach((node) => leaves.add(node))
+    let totalJsonSize = sizeof(json)
+
+    while (!leaves.empty() && jsonExceedMaxSize(totalJsonSize)) {
+        const curNode = leaves.poll()
+
+        const isDepthGreaterThanOne = curNode && curNode.depth > 1
+        const isTruncatable = curNode && (!nonTruncatableKeys.includes(curNode.key))
+
+        if (isDepthGreaterThanOne && isTruncatable) {
+            totalJsonSize += SIZE_OF_TRUNCATION_TEXT_PLACEHOLDER - curNode.size
+
+            const parent = curNode.parent
+
+            parent.value[curNode.key] = TRUNCATION_TEXT_PLACEHOLDER
+
+            nodes[parent.index].numberOfChildren--
+            if (nodes[parent.index].numberOfChildren == 0) {
+                leaves.add(nodes[parent.index])
+            }
+        }
+    }
+    return json as Record<string, StepOutput>
+}
+
+function traverseJsonAndConvertToNodes(root: unknown) {
+
+    const nodesQueue = new Queue<BfsNode>()
+    nodesQueue.enqueue({ key: '', value: root, parent: { index: -1, value: {} }, depth: 0 })
+
+    const nodes: Node[] = []
+
+    while (!nodesQueue.isEmpty()) {
+        const curNode = nodesQueue.dequeue()
+        const children = findChildren(curNode.value, curNode.key === 'iterations')
+
+        nodes.push({
+            index: nodes.length,
+            size: children.length === 0 ? sizeof(curNode.value) : children.length * SIZE_OF_TRUNCATION_TEXT_PLACEHOLDER,
+            key: curNode.key,
+            parent: {
+                index: curNode.parent.index,
+                value: curNode.parent.value as Record<Key, unknown>,
+            },
+            numberOfChildren: children.length,
+            depth: curNode.depth,
+        })
+
+        children.forEach((child) => {
+            const key = child[0], value = child[1]
+            nodesQueue.enqueue({ value, key, parent: { index: nodes.length - 1, value: curNode.value }, depth: curNode.depth + 1 })
+        })
+    }
+
+    return nodes
+}
+
+function findChildren(curNode: unknown, traverseArray: boolean): [Key, unknown][] {
+    if (isObject(curNode)) {
+        return Object.entries(curNode)
+    }
+    // Array should be treated as a leaf node as If it has too many small items, It will prioritize the other steps first 
+    if (Array.isArray(curNode) && traverseArray) {
+        const children: [Key, unknown][] = []
+        for (let i = 0; i < curNode.length; i++) {
+            children.push([i, curNode[i]])
+        }
+        return children
+    }
+    return []
+}
+
+const jsonExceedMaxSize = (jsonSize: number): boolean => {
+    return jsonSize > MAX_SIZE_FOR_ALL_ENTRIES
+}
+
+type Node = {
+    index: number
+    size: number
+    key: Key
+    parent: {
+        index: number
+        value: Record<Key, unknown>
+    }
+    numberOfChildren: number
+    depth: number
+}
+
+type BfsNode = {
+    value: unknown
+    key: Key
+    parent: {
+        index: number
+        value: unknown
+    }
+    depth: number
+}
+
+type Key = string | number | symbol

activepieces-fork/packages/engine/src/lib/helper/piece-helper.ts

@@ -0,0 +1,253 @@
+import {
+    DropdownProperty,
+    DynamicProperties,
+    ExecutePropsResult,
+    getAuthPropertyForValue,
+    MultiSelectDropdownProperty,
+    PieceAuthProperty,
+    PieceMetadata,
+    PiecePropertyMap,
+    pieceTranslation,
+    PropertyType,
+    StaticPropsValue } from '@activepieces/pieces-framework'
+import {
+    AppConnectionType,
+    AppConnectionValue,
+    EngineGenericError,
+    ExecuteExtractPieceMetadata,
+    ExecutePropsOptions,
+    ExecuteValidateAuthOperation,
+    ExecuteValidateAuthResponse,
+    isNil,
+} from '@activepieces/shared'
+import { EngineConstants } from '../handler/context/engine-constants'
+import { testExecutionContext } from '../handler/context/test-execution-context'
+import { createFlowsContext } from '../services/flows.service'
+import { utils } from '../utils'
+import { createPropsResolver } from '../variables/props-resolver'
+import { pieceLoader } from './piece-loader'
+
+export const pieceHelper = {
+    async executeProps( operation: ExecutePropsParams): Promise<ExecutePropsResult<PropertyType.DROPDOWN | PropertyType.MULTI_SELECT_DROPDOWN | PropertyType.DYNAMIC>> {
+        const constants = EngineConstants.fromExecutePropertyInput(operation)
+        const executionState = await testExecutionContext.stateFromFlowVersion({
+            apiUrl: operation.internalApiUrl,
+            flowVersion: operation.flowVersion,
+            projectId: operation.projectId,
+            engineToken: operation.engineToken,
+            sampleData: operation.sampleData,
+        })
+        const { property, piece } = await pieceLoader.getPropOrThrow({ pieceName: operation.pieceName, pieceVersion: operation.pieceVersion, actionOrTriggerName: operation.actionOrTriggerName, propertyName: operation.propertyName, devPieces: EngineConstants.DEV_PIECES })
+    
+        if (property.type !== PropertyType.DROPDOWN && property.type !== PropertyType.MULTI_SELECT_DROPDOWN && property.type !== PropertyType.DYNAMIC) {
+            throw new EngineGenericError('PropertyTypeNotExecutableError', `Property type is not executable: ${property.type} for ${property.displayName}`)
+        }
+        const { data: executePropsResult, error: executePropsError } = await utils.tryCatchAndThrowOnEngineError((async (): Promise<ExecutePropsResult<PropertyType.DROPDOWN | PropertyType.MULTI_SELECT_DROPDOWN | PropertyType.DYNAMIC>> => {
+            const { resolvedInput } = await createPropsResolver({
+                apiUrl: constants.internalApiUrl,
+                projectId: constants.projectId,
+                engineToken: constants.engineToken,
+                contextVersion: piece.getContextInfo?.().version,
+            }).resolve<
+            StaticPropsValue<PiecePropertyMap>
+            >({
+                unresolvedInput: operation.input,
+                executionState,
+            })
+            const ctx = {
+                searchValue: operation.searchValue,
+                server: {
+                    token: constants.engineToken,
+                    apiUrl: constants.internalApiUrl,
+                    publicUrl: operation.publicApiUrl,
+                },
+                project: {
+                    id: constants.projectId,
+                    externalId: constants.externalProjectId,
+                },
+                flows: createFlowsContext(constants),
+                step: {
+                    name: operation.actionOrTriggerName,
+                },
+                connections: utils.createConnectionManager({
+                    projectId: constants.projectId,
+                    engineToken: constants.engineToken,
+                    apiUrl: constants.internalApiUrl,
+                    target: 'properties',
+                    contextVersion: piece.getContextInfo?.().version,
+                }),
+            }
+          
+            switch (property.type) {
+                case PropertyType.DYNAMIC: {
+                    const dynamicProperty = property as DynamicProperties<boolean>
+                    const props = await dynamicProperty.props(resolvedInput, ctx)
+                    return {
+                        type: PropertyType.DYNAMIC,
+                        options: props,
+                    }
+                }
+                case PropertyType.MULTI_SELECT_DROPDOWN: {
+                    const multiSelectProperty = property as MultiSelectDropdownProperty<
+                    unknown,
+                    boolean
+                    >
+                    const options = await multiSelectProperty.options(resolvedInput, ctx)
+                    return {
+                        type: PropertyType.MULTI_SELECT_DROPDOWN,
+                        options,
+                    }
+                }
+                case PropertyType.DROPDOWN: {
+                    const dropdownProperty = property as DropdownProperty<unknown, boolean>
+                    const options = await dropdownProperty.options(resolvedInput, ctx)
+                    return {
+                        type: PropertyType.DROPDOWN,
+                        options,
+                    }
+                }
+                default: {
+                    throw new EngineGenericError('PropertyTypeNotExecutableError', `Property type is not executable: ${property}`)
+                }
+            }
+        }))
+        
+        if (executePropsError) {
+            console.error(executePropsError)
+            return {
+                type: property.type,
+                options: {
+                    disabled: true,
+                    options: [],
+                    placeholder: 'Throws an error, reconnect or refresh the page',
+                },
+            }
+        }
+
+        return executePropsResult
+    },
+
+    async executeValidateAuth(
+        { params, devPieces }: { params: ExecuteValidateAuthOperation, devPieces: string[] },
+    ): Promise<ExecuteValidateAuthResponse> {
+        const { piece: piecePackage } = params
+
+        const piece = await pieceLoader.loadPieceOrThrow({ pieceName: piecePackage.pieceName, pieceVersion: piecePackage.pieceVersion, devPieces })
+        const server = {
+            apiUrl: params.internalApiUrl.endsWith('/') ? params.internalApiUrl : params.internalApiUrl + '/',
+            publicUrl: params.publicApiUrl,
+        }
+        return  validateAuth({
+            authValue: params.auth,
+            pieceAuth: piece.auth,
+            server,
+        })
+
+    },
+
+    async extractPieceMetadata({ devPieces, params }: { devPieces: string[], params: ExecuteExtractPieceMetadata }): Promise<PieceMetadata> {
+        const { pieceName, pieceVersion } = params
+        const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces })
+        const pieceAlias = pieceLoader.getPackageAlias({ pieceName, pieceVersion, devPieces })
+        const pieceFolderPath = await pieceLoader.getPiecePath({ packageName: pieceAlias, devPieces })
+        const i18n = await pieceTranslation.initializeI18n(pieceFolderPath)
+        const fullMetadata = piece.metadata()
+        return {
+            ...fullMetadata,
+            name: pieceName,
+            version: pieceVersion,
+            authors: piece.authors,
+            i18n,
+        }
+    },
+}
+
+type ExecutePropsParams = Omit<ExecutePropsOptions, 'piece'> & { pieceName: string, pieceVersion: string }
+
+
+function mismatchAuthTypeErrorMessage(pieceAuthType: PropertyType, connectionType: AppConnectionType): ExecuteValidateAuthResponse {
+    return {
+        valid: false,
+        error: `Connection value type does not match piece auth type: ${pieceAuthType} !== ${connectionType}`,
+    }
+}
+
+const validateAuth = async ({
+    server,
+    authValue,
+    pieceAuth,
+}: ValidateAuthParams): Promise<ExecuteValidateAuthResponse> => {
+    if (isNil(pieceAuth)) {
+        return {
+            valid: true,
+        }
+    }
+    const usedPieceAuth = getAuthPropertyForValue({
+        authValueType: authValue.type,
+        pieceAuth,
+    })
+
+    if (isNil(usedPieceAuth)) {
+        return {
+            valid: false,
+            error: 'No piece auth found for auth value',
+        }
+    }
+    if (isNil(usedPieceAuth.validate)) {
+        return {
+            valid: true,
+        }
+    }
+  
+
+    switch (usedPieceAuth.type) {
+        case PropertyType.OAUTH2:{
+            if (authValue.type !== AppConnectionType.OAUTH2 && authValue.type !== AppConnectionType.CLOUD_OAUTH2 && authValue.type !== AppConnectionType.PLATFORM_OAUTH2) {
+                return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type)
+            }
+            return usedPieceAuth.validate({
+                auth: authValue,
+                server,
+            })
+        }
+        case PropertyType.BASIC_AUTH:{
+            if (authValue.type !== AppConnectionType.BASIC_AUTH) {
+                return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type)
+            }
+            return usedPieceAuth.validate({
+                auth: authValue,
+                server,
+            })
+        }
+        case PropertyType.SECRET_TEXT:{
+            if (authValue.type !== AppConnectionType.SECRET_TEXT) {
+                return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type)
+            }
+            return usedPieceAuth.validate({
+                auth: authValue.secret_text,
+                server,
+            })
+        }
+        case PropertyType.CUSTOM_AUTH:{
+            if (authValue.type !== AppConnectionType.CUSTOM_AUTH) {
+                return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type)
+            }
+            return usedPieceAuth.validate({
+                auth: authValue.props,
+                server,
+            })
+        }
+        default: {
+            throw new EngineGenericError('InvalidAuthTypeError', 'Invalid auth type')
+        }
+    }
+}
+
+type ValidateAuthParams = {
+    server: {
+        apiUrl: string
+        publicUrl: string
+    }
+    authValue: AppConnectionValue
+    pieceAuth: PieceAuthProperty | PieceAuthProperty[] | undefined
+}

activepieces-fork/packages/engine/src/lib/helper/piece-loader.ts

@@ -0,0 +1,208 @@
+import fs from 'fs/promises'
+import path from 'path'
+import { Action, Piece, PiecePropertyMap, Trigger } from '@activepieces/pieces-framework'
+import { ActivepiecesError, EngineGenericError, ErrorCode, extractPieceFromModule, getPackageAliasForPiece, getPieceNameFromAlias, isNil, trimVersionFromAlias } from '@activepieces/shared'
+import { utils } from '../utils'
+
+export const pieceLoader = {
+    loadPieceOrThrow: async (
+        { pieceName, pieceVersion, devPieces }: LoadPieceParams,
+    ): Promise<Piece> => {
+        const { data: piece, error: pieceError } = await utils.tryCatchAndThrowOnEngineError(async () => {
+            const packageName = pieceLoader.getPackageAlias({
+                pieceName,
+                pieceVersion,
+                devPieces,
+            })
+            const piecePath = await pieceLoader.getPiecePath({ packageName, devPieces })
+            const module = await import(piecePath)
+
+            const piece = extractPieceFromModule<Piece>({
+                module,
+                pieceName,
+                pieceVersion,
+            })
+
+            if (isNil(piece)) {
+                throw new EngineGenericError('PieceNotFoundError', `Piece not found for piece: ${pieceName}, pieceVersion: ${pieceVersion}`)
+            }
+            return piece
+        })
+        if (pieceError) {
+            throw pieceError
+        }
+        return piece
+    },
+
+    getPieceAndTriggerOrThrow: async (params: GetPieceAndTriggerParams): Promise<{ piece: Piece, pieceTrigger: Trigger }> => {
+        const { pieceName, pieceVersion, triggerName, devPieces } = params
+        const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces })
+        const trigger = piece.getTrigger(triggerName)
+
+        if (trigger === undefined) {
+            throw new EngineGenericError('TriggerNotFoundError', `Trigger not found, pieceName=${pieceName}, triggerName=${triggerName}`)
+        }
+
+        return {
+            piece,
+            pieceTrigger: trigger,
+        }
+    },
+
+    getPieceAndActionOrThrow: async (params: GetPieceAndActionParams): Promise<{ piece: Piece, pieceAction: Action }> => {
+        const { pieceName, pieceVersion, actionName, devPieces } = params
+
+        const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces })
+        const pieceAction = piece.getAction(actionName)
+
+        if (isNil(pieceAction)) {
+            throw new ActivepiecesError({
+                code: ErrorCode.STEP_NOT_FOUND,
+                params: {
+                    pieceName,
+                    pieceVersion,
+                    stepName: actionName,
+                },
+            })
+        }
+
+        return {
+            piece,
+            pieceAction,
+        }
+    },
+
+    getPropOrThrow: async ({ pieceName, pieceVersion, actionOrTriggerName, propertyName, devPieces }: GetPropParams) => {
+        const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces })
+
+        const actionOrTrigger = piece.getAction(actionOrTriggerName) ?? piece.getTrigger(actionOrTriggerName)
+
+        if (isNil(actionOrTrigger)) {
+            throw new ActivepiecesError({
+                code: ErrorCode.STEP_NOT_FOUND,
+                params: {
+                    pieceName,
+                    pieceVersion,
+                    stepName: actionOrTriggerName,
+                },
+            })
+        }
+
+        const property = (actionOrTrigger.props as PiecePropertyMap)[propertyName]
+
+        if (isNil(property)) {
+            throw new ActivepiecesError({
+                code: ErrorCode.CONFIG_NOT_FOUND,
+                params: {
+                    pieceName,
+                    pieceVersion,
+                    stepName: actionOrTriggerName,
+                    configName: propertyName,
+                },
+            })
+        }
+
+        return { property, piece }
+    },
+
+    getPackageAlias: ({ pieceName, pieceVersion, devPieces }: GetPackageAliasParams) => {
+        if (devPieces.includes(getPieceNameFromAlias(pieceName))) {
+            return pieceName
+        }
+
+        return getPackageAliasForPiece({
+            pieceName,
+            pieceVersion,
+        })
+    },
+
+    getPiecePath: async ({ packageName, devPieces }: GetPiecePathParams): Promise<string> => {
+        const piecePath = devPieces.includes(getPieceNameFromAlias(packageName)) 
+            ? await loadPieceFromDistFolder(packageName) 
+            : await traverseAllParentFoldersToFindPiece(packageName)
+        if (isNil(piecePath)) {
+            throw new EngineGenericError('PieceNotFoundError', `Piece not found for package: ${packageName}`)
+        }
+        return piecePath
+    },
+}
+
+async function loadPieceFromDistFolder(packageName: string): Promise<string | null> {
+    const distPath = path.resolve('dist/packages/pieces')
+    const entries = (await utils.walk(distPath)).filter((entry) => entry.name === 'package.json')
+    for (const entry of entries) {
+        const { data: packageJsonPath } = await utils.tryCatchAndThrowOnEngineError((async () => {
+            const packageJsonPath = entry.path
+            const packageJsonContent = await fs.readFile(packageJsonPath, 'utf-8')
+            const packageJson = JSON.parse(packageJsonContent)
+            if (packageJson.name === packageName) {
+                return path.dirname(packageJsonPath)
+            }
+            return null
+        }))
+        if (packageJsonPath) {
+            return packageJsonPath
+        }
+    }
+    return null
+}
+
+async function traverseAllParentFoldersToFindPiece(packageName: string): Promise<string | null> {
+    const rootDir = path.parse(__dirname).root
+    let currentDir = __dirname
+    const maxIterations = currentDir.split(path.sep).length
+    for (let i = 0; i < maxIterations; i++) {
+        const piecePath = path.resolve(currentDir, 'pieces', packageName, 'node_modules', trimVersionFromAlias(packageName))
+
+        if (await utils.folderExists(piecePath)) {
+            return piecePath
+        }
+
+        const parentDir = path.dirname(currentDir)
+        if (parentDir === currentDir || currentDir === rootDir) {
+            break
+        }
+        currentDir = parentDir
+    }
+    return null
+}
+
+type GetPiecePathParams = {
+    packageName: string
+    devPieces: string[]
+}
+
+type LoadPieceParams = {
+    pieceName: string
+    pieceVersion: string
+    devPieces: string[]
+}
+
+type GetPieceAndTriggerParams = {
+    pieceName: string
+    pieceVersion: string
+    triggerName: string
+    devPieces: string[]
+}
+
+type GetPieceAndActionParams = {
+    pieceName: string
+    pieceVersion: string
+    actionName: string
+    devPieces: string[]
+}
+
+type GetPropParams = {
+    pieceName: string
+    pieceVersion: string
+    actionOrTriggerName: string
+    propertyName: string
+    devPieces: string[]
+}
+
+type GetPackageAliasParams = {
+    pieceName: string
+    devPieces: string[]
+    pieceVersion: string
+}
+

activepieces-fork/packages/engine/src/lib/helper/trigger-helper.ts

@@ -0,0 +1,312 @@
+import { inspect } from 'node:util'
+import { PiecePropertyMap, StaticPropsValue, TriggerStrategy } from '@activepieces/pieces-framework'
+import { assertEqual, AUTHENTICATION_PROPERTY_NAME, EngineGenericError, EventPayload, ExecuteTriggerOperation, ExecuteTriggerResponse, FlowTrigger, InvalidCronExpressionError, isNil, PieceTrigger, PropertySettings, ScheduleOptions, TriggerHookType, TriggerSourceScheduleType } from '@activepieces/shared'
+import { isValidCron } from 'cron-validator'
+import { EngineConstants } from '../handler/context/engine-constants'
+import { FlowExecutorContext } from '../handler/context/flow-execution-context'
+import { createFlowsContext } from '../services/flows.service'
+import { createFilesService } from '../services/step-files.service'
+import { createContextStore } from '../services/storage.service'
+import { utils } from '../utils'
+import { propsProcessor } from '../variables/props-processor'
+import { createPropsResolver } from '../variables/props-resolver'
+import { pieceLoader } from './piece-loader'
+
+type Listener = {
+    events: string[]
+    identifierValue: string
+    identifierKey: string
+}
+
+export const triggerHelper = {
+    async executeOnStart(trigger: FlowTrigger, constants: EngineConstants, payload: unknown) {
+        const { pieceName, pieceVersion, triggerName, input, propertySettings } = (trigger as PieceTrigger).settings
+
+        if (isNil(triggerName)) {
+            throw new EngineGenericError('TriggerNameNotSetError', 'Trigger name is not set')
+        }
+
+        const { pieceTrigger, processedInput, piece } = await prepareTriggerExecution({
+            pieceName,
+            pieceVersion,
+            triggerName,
+            input,
+            projectId: constants.projectId,
+            apiUrl: constants.internalApiUrl,
+            engineToken: constants.engineToken,
+            devPieces: constants.devPieces,
+            propertySettings,
+        })
+        const isOldVersionOrNotSupported = isNil(pieceTrigger.onStart)
+        if (isOldVersionOrNotSupported) {
+            return
+        }
+        const context = {
+            store: createContextStore({
+                apiUrl: constants.internalApiUrl,
+                prefix: '',
+                flowId: constants.flowId,
+                engineToken: constants.engineToken,
+            }),
+            auth: processedInput[AUTHENTICATION_PROPERTY_NAME],
+            propsValue: processedInput,
+            payload,
+            run: {
+                id: constants.flowRunId,
+            },
+            step: {
+                name: triggerName,
+            },
+            project: {
+                id: constants.projectId,
+                externalId: constants.externalProjectId,
+            },
+            connections: utils.createConnectionManager({
+                apiUrl: constants.internalApiUrl,
+                projectId: constants.projectId,
+                engineToken: constants.engineToken,
+                target: 'triggers',
+                contextVersion: piece.getContextInfo?.().version,
+            }),
+        }
+        await pieceTrigger.onStart(context)
+    },
+
+    async executeTrigger({ params, constants }: ExecuteTriggerParams): Promise<ExecuteTriggerResponse<TriggerHookType>> {
+        const { pieceName, pieceVersion, triggerName, input, propertySettings } = (params.flowVersion.trigger as PieceTrigger).settings
+
+        if (isNil(triggerName)) {
+            throw new EngineGenericError('TriggerNameNotSetError', 'Trigger name is not set')
+        }
+
+        const { piece, pieceTrigger, processedInput } = await prepareTriggerExecution({
+            pieceName,
+            pieceVersion,
+            triggerName,
+            input,
+            projectId: params.projectId,
+            apiUrl: constants.internalApiUrl,
+            engineToken: params.engineToken,
+            devPieces: constants.devPieces,
+            propertySettings,
+        })
+
+        const appListeners: Listener[] = []
+        const prefix = params.test ? 'test' : ''
+        let scheduleOptions: ScheduleOptions | undefined = undefined
+        const context = {
+            store: createContextStore({
+                apiUrl: constants.internalApiUrl,
+                prefix,
+                flowId: params.flowVersion.flowId,
+                engineToken: params.engineToken,
+            }),
+            step: {
+                name: triggerName,
+            },
+            app: {
+                createListeners({ events, identifierKey, identifierValue }: Listener): void {
+                    appListeners.push({ events, identifierValue, identifierKey })
+                },
+            },
+            setSchedule(request: ScheduleOptions) {
+                if (!isValidCron(request.cronExpression)) {
+                    throw new InvalidCronExpressionError(request.cronExpression)
+                }
+                scheduleOptions = {
+                    type: TriggerSourceScheduleType.CRON_EXPRESSION,
+                    cronExpression: request.cronExpression,
+                    timezone: request.timezone ?? 'UTC',
+                }
+            },
+            flows: createFlowsContext({
+                engineToken: params.engineToken,
+                internalApiUrl: constants.internalApiUrl,
+                flowId: params.flowVersion.flowId,
+                flowVersionId: params.flowVersion.id,
+            }),
+            webhookUrl: params.webhookUrl,
+            auth: processedInput[AUTHENTICATION_PROPERTY_NAME],
+            propsValue: processedInput,
+            payload: params.triggerPayload ?? {},
+            project: {
+                id: params.projectId,
+                externalId: constants.externalProjectId,
+            },
+            server: {
+                token: params.engineToken,
+                apiUrl: constants.internalApiUrl,
+                publicUrl: params.publicApiUrl,
+            },
+            connections: utils.createConnectionManager({
+                apiUrl: constants.internalApiUrl,
+                projectId: constants.projectId,
+                engineToken: constants.engineToken,
+                target: 'triggers',
+                contextVersion: piece.getContextInfo?.().version,
+            }),
+        }
+        switch (params.hookType) {
+            case TriggerHookType.ON_DISABLE: {
+                await pieceTrigger.onDisable(context)
+                return {}
+            }
+            case TriggerHookType.ON_ENABLE: {
+                await pieceTrigger.onEnable(context)
+                return {
+                    listeners: appListeners,
+                    scheduleOptions: pieceTrigger.type === TriggerStrategy.POLLING ? scheduleOptions : undefined,
+                }
+            }
+            case TriggerHookType.RENEW: {
+                assertEqual(pieceTrigger.type, TriggerStrategy.WEBHOOK, 'triggerType', 'WEBHOOK')
+                await pieceTrigger.onRenew(context)
+                return {
+                    success: true,
+                }
+            }
+            case TriggerHookType.HANDSHAKE: {
+                const { data: handshakeResponse, error: handshakeResponseError } = await utils.tryCatchAndThrowOnEngineError(() => pieceTrigger.onHandshake(context))
+
+                if (handshakeResponseError) {
+                    console.error(handshakeResponseError)
+                    return {
+                        success: false,
+                        message: `Error while testing trigger: ${inspect(handshakeResponseError)}`,
+                    }
+                }
+                return {
+                    success: true,
+                    response: handshakeResponse,
+                }
+            }
+            case TriggerHookType.TEST: {
+                const { data: testResponse, error: testResponseError } = await utils.tryCatchAndThrowOnEngineError(() => pieceTrigger.test({
+                    ...context,
+                    files: createFilesService({
+                        apiUrl: constants.internalApiUrl,
+                        engineToken: params.engineToken!,
+                        stepName: triggerName,
+                        flowId: params.flowVersion.flowId,
+                    }),
+                }))
+
+                if (testResponseError) {
+                    console.error(testResponseError)
+                    return {
+                        success: false,
+                        message: `Error while testing trigger: ${inspect(testResponseError)}`,
+                        output: [],
+                    }
+                }
+                return {
+                    success: true,
+                    output: testResponse,
+                }
+            }
+            case TriggerHookType.RUN: {
+                if (pieceTrigger.type === TriggerStrategy.APP_WEBHOOK) {
+
+                    const { data: verified, error: verifiedError } = await utils.tryCatchAndThrowOnEngineError(async () => {
+                        if (!params.appWebhookUrl) {
+                            throw new EngineGenericError('AppWebhookUrlNotAvailableError', `App webhook url is not available for piece name ${pieceName}`)
+                        }
+                        if (!params.webhookSecret) {
+                            throw new EngineGenericError('WebhookSecretNotAvailableError', `Webhook secret is not available for piece name ${pieceName}`)
+                        }
+
+                        return piece.events?.verify({
+                            appWebhookUrl: params.appWebhookUrl,
+                            payload: params.triggerPayload as EventPayload,
+                            webhookSecret: params.webhookSecret,
+                        })
+                    })
+
+                    if (verifiedError) {
+                        return {
+                            success: false,
+                            message: `Error while verifying webhook: ${inspect(verifiedError)}`,
+                            output: [],
+                        }
+                    }
+                    if (isNil(verified)) {
+                        return {
+                            success: false,
+                            message: 'Webhook is not verified',
+                            output: [],
+                        }
+                    }
+                }
+
+                const { data: triggerRunResult, error: triggerRunError } = await utils.tryCatchAndThrowOnEngineError(async () => {
+                    const items = await pieceTrigger.run({
+                        ...context,
+                        files: createFilesService({
+                            apiUrl: constants.internalApiUrl,
+                            engineToken: params.engineToken!,
+                            flowId: params.flowVersion.flowId,
+                            stepName: triggerName,
+                        }),
+                    })
+                    return {
+                        success: true,
+                        output: items,
+                    }
+                })
+
+                if (triggerRunError) {
+                    return {
+                        success: false,
+                        message: triggerRunError.message,
+                        output: [],
+                    }
+                }
+                return triggerRunResult
+            }
+        }
+    },
+}
+
+type ExecuteTriggerParams = {
+    params: ExecuteTriggerOperation<TriggerHookType>
+    constants: EngineConstants
+}
+
+async function prepareTriggerExecution({ pieceName, pieceVersion, triggerName, input, propertySettings, projectId, apiUrl, engineToken, devPieces }: PrepareTriggerExecutionParams) {
+    const { piece, pieceTrigger } = await pieceLoader.getPieceAndTriggerOrThrow({
+        pieceName,
+        pieceVersion,
+        triggerName,
+        devPieces,
+    })
+
+    const { resolvedInput } = await createPropsResolver({
+        apiUrl,
+        projectId,
+        engineToken,
+        contextVersion: piece.getContextInfo?.().version,
+    }).resolve<StaticPropsValue<PiecePropertyMap>>({
+        unresolvedInput: input,
+        executionState: FlowExecutorContext.empty(),
+    })
+
+    const { processedInput, errors } = await propsProcessor.applyProcessorsAndValidators(resolvedInput, pieceTrigger.props, piece.auth, pieceTrigger.requireAuth, propertySettings)
+
+    if (Object.keys(errors).length > 0) {
+        throw new Error(JSON.stringify(errors, null, 2))
+    }
+
+    return { piece, pieceTrigger, processedInput }
+}
+
+type PrepareTriggerExecutionParams = {
+    pieceName: string
+    pieceVersion: string
+    triggerName: string
+    input: unknown
+    propertySettings: Record<string, PropertySettings>
+    projectId: string
+    apiUrl: string
+    engineToken: string
+    devPieces: string[]
+}

activepieces-fork/packages/engine/src/lib/operations/auth-validation.operation.ts

@@ -0,0 +1,23 @@
+import {
+    EngineResponse,
+    EngineResponseStatus,
+    ExecuteValidateAuthOperation,
+    ExecuteValidateAuthResponse,
+} from '@activepieces/shared'
+import { EngineConstants } from '../handler/context/engine-constants'
+import { pieceHelper } from '../helper/piece-helper'
+
+export const authValidationOperation = {
+    execute: async (operation: ExecuteValidateAuthOperation): Promise<EngineResponse<ExecuteValidateAuthResponse>> => {
+        const input = operation as ExecuteValidateAuthOperation
+        const output = await pieceHelper.executeValidateAuth({
+            params: input,
+            devPieces: EngineConstants.DEV_PIECES,
+        })
+
+        return {
+            status: EngineResponseStatus.OK,
+            response: output,
+        }
+    },
+}

activepieces-fork/packages/engine/src/lib/operations/flow.operation.ts

@@ -0,0 +1,135 @@
+import {
+    BeginExecuteFlowOperation,
+    EngineResponse,
+    EngineResponseStatus,
+    ExecuteFlowOperation,
+    ExecuteTriggerResponse,
+    ExecutionType,
+    FlowActionType,
+    flowStructureUtil,
+    GenericStepOutput,
+    isNil,
+    LoopStepOutput,
+    StepOutput,
+    StepOutputStatus,
+    TriggerHookType,
+    TriggerPayload,
+} from '@activepieces/shared'
+import { EngineConstants } from '../handler/context/engine-constants'
+import { FlowExecutorContext } from '../handler/context/flow-execution-context'
+import { testExecutionContext } from '../handler/context/test-execution-context'
+import { flowExecutor } from '../handler/flow-executor'
+import { triggerHelper } from '../helper/trigger-helper'
+import { progressService } from '../services/progress.service'
+
+export const flowOperation = {
+    execute: async (operation: ExecuteFlowOperation): Promise<EngineResponse<undefined>> => {
+        const input = operation as ExecuteFlowOperation
+        const constants = EngineConstants.fromExecuteFlowInput(input)
+        const output: FlowExecutorContext = (await executieSingleStepOrFlowOperation(input)).finishExecution()
+        await progressService.sendUpdate({
+            engineConstants: constants,
+            flowExecutorContext: output,
+            updateImmediate: true,
+        })
+        return {
+            status: EngineResponseStatus.OK,
+            response: undefined,
+            delayInSeconds: output.getDelayedInSeconds(),
+        }
+    },
+}
+
+const executieSingleStepOrFlowOperation = async (input: ExecuteFlowOperation): Promise<FlowExecutorContext> => {
+    const constants = EngineConstants.fromExecuteFlowInput(input)
+    const testSingleStepMode = !isNil(constants.stepNameToTest)
+    if (testSingleStepMode) {
+        const testContext = await testExecutionContext.stateFromFlowVersion({
+            apiUrl: input.internalApiUrl,
+            flowVersion: input.flowVersion,
+            excludedStepName: input.stepNameToTest!,
+            projectId: input.projectId,
+            engineToken: input.engineToken,
+            sampleData: input.sampleData,
+        })
+        const step = flowStructureUtil.getActionOrThrow(input.stepNameToTest!, input.flowVersion.trigger)
+        return flowExecutor.execute({
+            action: step,
+            executionState: await getFlowExecutionState(input, testContext),
+            constants: EngineConstants.fromExecuteFlowInput(input),
+        })
+    }
+    return flowExecutor.executeFromTrigger({
+        executionState: await getFlowExecutionState(input, FlowExecutorContext.empty()),
+        constants,
+        input,
+    })
+}
+
+async function getFlowExecutionState(input: ExecuteFlowOperation, flowContext: FlowExecutorContext): Promise<FlowExecutorContext> {
+    switch (input.executionType) {
+        case ExecutionType.BEGIN: {
+            const newPayload = await runOrReturnPayload(input)
+            flowContext = flowContext.upsertStep(input.flowVersion.trigger.name, GenericStepOutput.create({
+                type: input.flowVersion.trigger.type,
+                status: StepOutputStatus.SUCCEEDED,
+                input: {},
+            }).setOutput(newPayload))
+            break
+        }
+        case ExecutionType.RESUME: {
+            break
+        }
+    }
+
+    for (const [step, output] of Object.entries(input.executionState.steps)) {
+        if ([StepOutputStatus.SUCCEEDED, StepOutputStatus.PAUSED].includes(output.status)) {
+            const newOutput = await insertSuccessStepsOrPausedRecursively(output)
+            if (!isNil(newOutput)) {
+                flowContext = flowContext.upsertStep(step, newOutput)
+            }
+        }
+    }
+    return flowContext
+}
+
+async function runOrReturnPayload(input: BeginExecuteFlowOperation): Promise<TriggerPayload> {
+    if (!input.executeTrigger) {
+        return input.triggerPayload as TriggerPayload
+    }
+    const newPayload = await triggerHelper.executeTrigger({
+        params: {
+            ...input,
+            hookType: TriggerHookType.RUN,
+            test: false,
+            webhookUrl: '',
+            triggerPayload: input.triggerPayload as TriggerPayload,
+        },
+        constants: EngineConstants.fromExecuteFlowInput(input),
+    }) as ExecuteTriggerResponse<TriggerHookType.RUN>
+    return newPayload.output[0] as TriggerPayload
+}
+
+
+async function insertSuccessStepsOrPausedRecursively(stepOutput: StepOutput): Promise<StepOutput | null> {
+    if (![StepOutputStatus.SUCCEEDED, StepOutputStatus.PAUSED].includes(stepOutput.status)) {
+        return null
+    }
+    if (stepOutput.type === FlowActionType.LOOP_ON_ITEMS) {
+        const loopOutput = new LoopStepOutput(stepOutput)
+        const iterations = loopOutput.output?.iterations ?? []
+        const newIterations: Record<string, StepOutput>[] = []
+        for (const iteration of iterations) {
+            const newSteps: Record<string, StepOutput> = {}
+            for (const [step, output] of Object.entries(iteration)) {
+                const newOutput = await insertSuccessStepsOrPausedRecursively(output)
+                if (!isNil(newOutput)) {
+                    newSteps[step] = newOutput
+                }
+            }
+            newIterations.push(newSteps)
+        }
+        return loopOutput.setIterations(newIterations)
+    }
+    return stepOutput
+}

activepieces-fork/packages/engine/src/lib/operations/index.ts

@@ -0,0 +1,42 @@
+import {
+    EngineOperation,
+    EngineOperationType,
+    EngineResponse,
+    ExecuteExtractPieceMetadataOperation,
+    ExecuteFlowOperation,
+    ExecutePropsOptions,
+    ExecuteTriggerOperation,    
+    ExecuteValidateAuthOperation,
+    ExecutionError,
+    ExecutionErrorType,
+    TriggerHookType, 
+} from '@activepieces/shared'
+import { authValidationOperation } from './auth-validation.operation'
+import { flowOperation } from './flow.operation'
+import { pieceMetadataOperation } from './piece-metadata.operation'
+import { propertyOperation } from './property.operation'
+import { triggerHookOperation } from './trigger-hook.operation'
+
+
+export async function execute(operationType: EngineOperationType, operation: EngineOperation): Promise<EngineResponse<unknown>> {
+    switch (operationType) {
+        case EngineOperationType.EXTRACT_PIECE_METADATA: {
+            return pieceMetadataOperation.extract(operation as ExecuteExtractPieceMetadataOperation)
+        }
+        case EngineOperationType.EXECUTE_FLOW: {
+            return flowOperation.execute(operation as ExecuteFlowOperation)
+        }
+        case EngineOperationType.EXECUTE_PROPERTY: {
+            return propertyOperation.execute(operation as ExecutePropsOptions)
+        }
+        case EngineOperationType.EXECUTE_TRIGGER_HOOK: {
+            return triggerHookOperation.execute(operation as ExecuteTriggerOperation<TriggerHookType>)
+        }
+        case EngineOperationType.EXECUTE_VALIDATE_AUTH: {
+            return authValidationOperation.execute(operation as ExecuteValidateAuthOperation)
+        }
+        default: {
+            throw new ExecutionError('Unsupported operation type', `Unsupported operation type: ${operationType}`, ExecutionErrorType.ENGINE)
+        }
+    }
+}

activepieces-fork/packages/engine/src/lib/operations/piece-metadata.operation.ts

@@ -0,0 +1,23 @@
+import { PieceMetadata } from '@activepieces/pieces-framework'
+import {
+    EngineResponse,
+    EngineResponseStatus,
+    ExecuteExtractPieceMetadataOperation,
+} from '@activepieces/shared'
+import { EngineConstants } from '../handler/context/engine-constants'
+import { pieceHelper } from '../helper/piece-helper'
+
+
+export const pieceMetadataOperation = {
+    extract: async (operation: ExecuteExtractPieceMetadataOperation): Promise<EngineResponse<PieceMetadata>>  => {
+        const input = operation as ExecuteExtractPieceMetadataOperation
+        const output = await pieceHelper.extractPieceMetadata({
+            params: input,
+            devPieces: EngineConstants.DEV_PIECES,
+        })
+        return {
+            status: EngineResponseStatus.OK,
+            response: output,
+        }
+    },
+}

activepieces-fork/packages/engine/src/lib/operations/property.operation.ts

@@ -0,0 +1,22 @@
+import { ExecutePropsResult, PropertyType } from '@activepieces/pieces-framework'
+import {
+    EngineResponse,
+    EngineResponseStatus,
+    ExecutePropsOptions,
+} from '@activepieces/shared'
+import { pieceHelper } from '../helper/piece-helper'
+
+
+export const propertyOperation = {
+    execute: async (operation: ExecutePropsOptions): Promise<EngineResponse<ExecutePropsResult<PropertyType.DROPDOWN | PropertyType.MULTI_SELECT_DROPDOWN | PropertyType.DYNAMIC>>> => {
+        const output = await pieceHelper.executeProps({
+            ...operation,
+            pieceName: operation.piece.pieceName,
+            pieceVersion: operation.piece.pieceVersion,
+        })
+        return {
+            status: EngineResponseStatus.OK,
+            response: output,
+        }
+    },
+}

activepieces-fork/packages/engine/src/lib/operations/trigger-hook.operation.ts

@@ -0,0 +1,24 @@
+import {
+    EngineResponse,
+    EngineResponseStatus,
+    ExecuteTriggerOperation,
+    ExecuteTriggerResponse,
+    TriggerHookType,
+} from '@activepieces/shared'
+import { EngineConstants } from '../handler/context/engine-constants'
+import { triggerHelper } from '../helper/trigger-helper'
+
+
+export const triggerHookOperation = {
+    execute: async (operation: ExecuteTriggerOperation<TriggerHookType>): Promise<EngineResponse<ExecuteTriggerResponse<TriggerHookType>>> => {
+        const input = operation as ExecuteTriggerOperation<TriggerHookType>
+        const output = await triggerHelper.executeTrigger({
+            params: input,
+            constants: EngineConstants.fromExecuteTriggerInput(input),
+        })
+        return {
+            status: EngineResponseStatus.OK,
+            response: output,
+        }
+    },
+}

activepieces-fork/packages/engine/src/lib/services/connections.service.ts

@@ -0,0 +1,99 @@
+import { ContextVersion } from '@activepieces/pieces-framework'
+import { AppConnection, AppConnectionStatus, AppConnectionType, AppConnectionValue, ConnectionExpiredError, ConnectionLoadingError, ConnectionNotFoundError, ExecutionError, FetchError } from '@activepieces/shared'
+import { StatusCodes } from 'http-status-codes'
+import { utils } from '../utils'
+    
+export const createConnectionService = ({ projectId, engineToken, apiUrl, contextVersion }: CreateConnectionServiceParams): ConnectionService => {
+    return {
+        async obtain(externalId: string): Promise<AppConnectionValue> {
+            const url = `${apiUrl}v1/worker/app-connections/${encodeURIComponent(externalId)}?projectId=${projectId}`
+
+            const { data: connectionValue, error: connectionValueError } = await utils.tryCatchAndThrowOnEngineError((async () => {
+                const response = await fetch(url, {
+                    method: 'GET',
+                    headers: {
+                        Authorization: `Bearer ${engineToken}`,
+                    },
+                })
+
+                if (!response.ok) {
+                    return handleResponseError({
+                        externalId,
+                        httpStatus: response.status,
+                    })
+                }
+                const connection: AppConnection = await response.json()
+                if (connection.status === AppConnectionStatus.ERROR) {
+                    throw new ConnectionExpiredError(externalId)
+                }
+                return getConnectionValue(connection, contextVersion)
+            }))
+            
+            if (connectionValueError) {
+                if (connectionValueError instanceof ExecutionError) {
+                    throw connectionValueError
+                }
+                return handleFetchError({
+                    url,
+                    cause: connectionValueError,
+                })
+            }
+            return connectionValue
+        },
+    }
+}
+
+const handleResponseError = ({ externalId, httpStatus }: HandleResponseErrorParams): never => {
+    if (httpStatus === StatusCodes.NOT_FOUND.valueOf()) {
+        throw new ConnectionNotFoundError(externalId)
+    }
+
+    throw new ConnectionLoadingError(externalId)
+}
+
+const handleFetchError = ({ url, cause }: HandleFetchErrorParams): never => {
+    throw new FetchError(url, cause)
+}
+
+const getConnectionValue = (connection: AppConnection, contextVersion: ContextVersion | undefined): AppConnectionValue => {
+    switch (contextVersion) {
+        case undefined:
+            return makeConnectionValueCompatibleWithContextV0(connection)
+        case ContextVersion.V1:
+            return connection.value
+        default:
+            return connection.value
+    }
+}
+
+function makeConnectionValueCompatibleWithContextV0(connection: AppConnection): AppConnectionValue {
+    switch (connection.value.type) {
+        case AppConnectionType.SECRET_TEXT:
+            return connection.value.secret_text as unknown as AppConnectionValue
+
+        case AppConnectionType.CUSTOM_AUTH:
+            return connection.value.props as unknown as AppConnectionValue
+        default:
+            return connection.value as unknown as AppConnectionValue
+    }
+}
+type ConnectionService = {
+    obtain(externalId: string): Promise<AppConnectionValue>
+}
+
+type CreateConnectionServiceParams = {
+    projectId: string
+    apiUrl: string
+    engineToken: string
+    contextVersion: ContextVersion | undefined
+}
+
+type HandleResponseErrorParams = {
+    externalId: string
+    httpStatus: number
+}
+
+type HandleFetchErrorParams = {
+    url: string
+    cause: unknown
+}

activepieces-fork/packages/engine/src/lib/services/flows.service.ts

@@ -0,0 +1,34 @@
+import { FlowsContext, ListFlowsContextParams } from '@activepieces/pieces-framework'
+import { PopulatedFlow, SeekPage } from '@activepieces/shared'
+
+
+type CreateFlowsServiceParams = {
+    engineToken: string
+    internalApiUrl: string
+    flowId: string
+    flowVersionId: string
+}
+export const createFlowsContext = ({ engineToken, internalApiUrl, flowId, flowVersionId }: CreateFlowsServiceParams): FlowsContext => {
+    return {
+        async list(params: ListFlowsContextParams): Promise<SeekPage<PopulatedFlow>> {
+            const queryParams = new URLSearchParams()
+            if (params?.externalIds) {
+                queryParams.set('externalIds', params.externalIds.join(','))
+            }
+            const response = await fetch(`${internalApiUrl}v1/engine/populated-flows?${queryParams.toString()}`, {
+                method: 'GET',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${engineToken}`,
+                },
+            })
+            return response.json()
+        },
+        current: {
+            id: flowId,
+            version: {
+                id: flowVersionId,
+            },
+        },
+    }
+}

activepieces-fork/packages/engine/src/lib/services/progress.service.ts

@@ -0,0 +1,196 @@
+import { OutputContext } from '@activepieces/pieces-framework'
+import { DEFAULT_MCP_DATA, EngineGenericError, EngineSocketEvent, FlowActionType, FlowRunStatus, GenericStepOutput, isFlowRunStateTerminal, isNil, logSerializer, StepOutput, StepOutputStatus, StepRunResponse, UpdateRunProgressRequest } from '@activepieces/shared'
+import { Mutex } from 'async-mutex'
+import dayjs from 'dayjs'
+import fetchRetry from 'fetch-retry'
+import { EngineConstants } from '../handler/context/engine-constants'
+import { FlowExecutorContext } from '../handler/context/flow-execution-context'
+import { utils } from '../utils'
+import { workerSocket } from '../worker-socket'
+
+
+let lastScheduledUpdateId: NodeJS.Timeout | null = null
+let lastActionExecutionTime: number | undefined = undefined
+let isGraceShutdownSignalReceived = false
+const MAXIMUM_UPDATE_THRESHOLD = 15000
+const DEBOUNCE_THRESHOLD = 5000
+const lock = new Mutex()
+const updateLock = new Mutex()
+const fetchWithRetry = fetchRetry(global.fetch)
+
+process.on('SIGTERM', () => {
+    isGraceShutdownSignalReceived = true
+})
+
+process.on('SIGINT', () => {
+    isGraceShutdownSignalReceived = true
+})
+
+export const progressService = {
+    sendUpdate: async (params: UpdateStepProgressParams): Promise<void> => {
+        return updateLock.runExclusive(async () => {
+            if (lastScheduledUpdateId) {
+                clearTimeout(lastScheduledUpdateId)
+            }
+
+            const shouldUpdateNow = isNil(lastActionExecutionTime) || (Date.now() - lastActionExecutionTime > MAXIMUM_UPDATE_THRESHOLD) || isGraceShutdownSignalReceived
+            if (shouldUpdateNow || params.updateImmediate) {
+                await sendUpdateRunRequest(params)
+                return
+            }
+
+            lastScheduledUpdateId = setTimeout(async () => {
+                await sendUpdateRunRequest(params)
+            }, DEBOUNCE_THRESHOLD)
+        })
+    },
+    createOutputContext: (params: CreateOutputContextParams): OutputContext => {
+        const { engineConstants, flowExecutorContext, stepName, stepOutput } = params
+        return {
+            update: async (params: { data: unknown }) => {
+                const trimmedSteps = await flowExecutorContext
+                    .upsertStep(stepName, stepOutput.setOutput(params.data))
+                    .trimmedSteps()
+                const stepResponse = extractStepResponse({
+                    steps: trimmedSteps,
+                    runId: engineConstants.flowRunId,
+                    stepName,
+                })
+                await workerSocket.sendToWorkerWithAck(EngineSocketEvent.UPDATE_STEP_PROGRESS, {
+                    projectId: engineConstants.projectId,
+                    stepResponse,
+                })
+            },
+        }
+    },
+}
+
+type CreateOutputContextParams = {
+    engineConstants: EngineConstants
+    flowExecutorContext: FlowExecutorContext
+    stepName: string
+    stepOutput: GenericStepOutput<FlowActionType.PIECE, unknown>
+}
+
+const queueUpdates: UpdateStepProgressParams[] = []
+
+const sendUpdateRunRequest = async (updateParams: UpdateStepProgressParams): Promise<void> => {
+    const isRunningMcp = updateParams.engineConstants.flowRunId === DEFAULT_MCP_DATA.flowRunId
+    if (updateParams.engineConstants.isRunningApTests || isRunningMcp) {
+        return
+    }
+    queueUpdates.push(updateParams)
+    await lock.runExclusive(async () => {
+        const params = queueUpdates.pop()
+        while (queueUpdates.length > 0) {
+            queueUpdates.pop()
+        }
+        if (isNil(params)) {
+            return
+        }
+        lastActionExecutionTime = Date.now()
+        const { flowExecutorContext, engineConstants } = params
+        const trimmedSteps = await flowExecutorContext.trimmedSteps()
+        const executionState = await logSerializer.serialize({
+            executionState: {
+                steps: trimmedSteps,
+            },
+        })
+        if (isNil(engineConstants.logsUploadUrl)) {
+            throw new EngineGenericError('LogsUploadUrlNotSetError', 'Logs upload URL is not set')
+        }
+        const uploadLogResponse = await uploadExecutionState(engineConstants.logsUploadUrl, executionState)
+        if (!uploadLogResponse.ok) {
+            throw new EngineGenericError('ProgressUpdateError', 'Failed to upload execution state', uploadLogResponse)
+        }
+
+        const stepResponse = extractStepResponse({
+            steps: trimmedSteps,
+            runId: engineConstants.flowRunId,
+            stepName: engineConstants.stepNameToTest,
+        })
+
+        const request: UpdateRunProgressRequest = {
+            runId: engineConstants.flowRunId,
+            projectId: engineConstants.projectId,
+            workerHandlerId: engineConstants.serverHandlerId ?? null,
+            httpRequestId: engineConstants.httpRequestId ?? null,
+            status: flowExecutorContext.verdict.status,
+            progressUpdateType: engineConstants.progressUpdateType,
+            logsFileId: engineConstants.logsFileId,
+            failedStep: flowExecutorContext.verdict.status === FlowRunStatus.FAILED ? flowExecutorContext.verdict.failedStep : undefined,
+            stepNameToTest: engineConstants.stepNameToTest,
+            stepResponse,
+            pauseMetadata: flowExecutorContext.verdict.status === FlowRunStatus.PAUSED ? flowExecutorContext.verdict.pauseMetadata : undefined,
+            finishTime: isFlowRunStateTerminal({
+                status: flowExecutorContext.verdict.status,
+                ignoreInternalError: false,
+            }) ? dayjs().toISOString() : undefined,
+            tags: Array.from(flowExecutorContext.tags),
+            stepsCount: flowExecutorContext.stepsCount,
+        }
+
+   
+        await sendProgressUpdate(request)
+
+    })
+}
+
+const sendProgressUpdate = async (request: UpdateRunProgressRequest): Promise<void> => {
+    const result = await utils.tryCatchAndThrowOnEngineError(() => 
+        workerSocket.sendToWorkerWithAck(EngineSocketEvent.UPDATE_RUN_PROGRESS, request),
+    )
+    if (result.error) {
+        throw new EngineGenericError('ProgressUpdateError', 'Failed to send progress update', result.error)
+    }
+}
+
+const uploadExecutionState = async (uploadUrl: string, executionState: Buffer, followRedirects = true): Promise<Response> => {
+    const response = await fetchWithRetry(uploadUrl, {
+        method: 'PUT',
+        body: new Uint8Array(executionState),
+        headers: {
+            'Content-Type': 'application/octet-stream',
+        },
+        redirect: 'manual',
+        retries: 3,
+        retryDelay: 3000,
+    })
+
+    if (followRedirects && response.status >= 300 && response.status < 400) {
+        const location = response.headers.get('location')!
+        return uploadExecutionState(location, executionState, false)
+    }
+    return response
+}
+
+
+const extractStepResponse = (params: ExtractStepResponse): StepRunResponse | undefined => {
+    if (isNil(params.stepName)) {
+        return undefined
+    }
+
+    const stepOutput = params.steps?.[params.stepName]
+    const isSuccess = stepOutput?.status === StepOutputStatus.SUCCEEDED || stepOutput?.status === StepOutputStatus.PAUSED
+    return {
+        runId: params.runId,
+        success: isSuccess,
+        input: stepOutput?.input,
+        output: stepOutput?.output,
+        standardError: isSuccess ? '' : (stepOutput?.errorMessage as string),
+        standardOutput: '',
+    }
+}
+
+
+type UpdateStepProgressParams = {
+    engineConstants: EngineConstants
+    flowExecutorContext: FlowExecutorContext
+    updateImmediate?: boolean
+}
+
+type ExtractStepResponse = {
+    steps: Record<string, StepOutput>
+    runId: string
+    stepName?: string
+}

activepieces-fork/packages/engine/src/lib/services/step-files.service.ts

@@ -0,0 +1,95 @@
+import { FilesService } from '@activepieces/pieces-framework'
+import { FileLocation, FileSizeError, FileStoreError, isNil, StepFileUpsertResponse } from '@activepieces/shared'
+import fetchRetry from 'fetch-retry'
+
+const MAX_FILE_SIZE_MB = Number(process.env.AP_MAX_FILE_SIZE_MB)
+const FILE_STORAGE_LOCATION = process.env.AP_FILE_STORAGE_LOCATION as FileLocation
+const USE_SIGNED_URL = (process.env.AP_S3_USE_SIGNED_URLS === 'true') && FILE_STORAGE_LOCATION === FileLocation.S3
+
+export type DefaultFileSystem = 'db' | 'local'
+
+type CreateFilesServiceParams = { apiUrl: string, stepName: string, flowId: string, engineToken: string }
+
+export function createFilesService({ stepName, flowId, engineToken, apiUrl }: CreateFilesServiceParams): FilesService {
+    return {
+        write: async ({ fileName, data }: { fileName: string, data: Buffer }): Promise<string> => {
+            validateFileSize(data)
+            const formData = createFormData({ fileName, data, stepName, flowId })
+            const result = await uploadFileMetadata({ formData, engineToken, apiUrl })
+            if (USE_SIGNED_URL) {
+                if (isNil(result.uploadUrl)) {
+                    throw new FileStoreError({
+                        status: 500,
+                        body: 'Upload URL is not available',
+                    })
+                }
+                await uploadFileContent({ url: result.uploadUrl, data })
+            }
+
+            return result.url
+        },
+    }
+}
+
+function validateFileSize(data: Buffer): void {
+    const maximumFileSizeInBytes = MAX_FILE_SIZE_MB * 1024 * 1024
+    if (data.length > maximumFileSizeInBytes) {
+        throw new FileSizeError(data.length / 1024 / 1024, MAX_FILE_SIZE_MB)
+    }
+}
+
+function createFormData({ fileName, data, stepName, flowId }: { fileName: string, data: Buffer, stepName: string, flowId: string }): FormData {
+    const formData = new FormData()
+    formData.append('stepName', stepName)
+    formData.append('flowId', flowId)
+    formData.append('contentLength', data.length.toString())
+    formData.append('fileName', fileName)
+
+    if (!USE_SIGNED_URL) {
+        formData.append('file', new Blob([data], { type: 'application/octet-stream' }), fileName)
+    }
+
+    return formData
+}
+
+async function uploadFileMetadata({ formData, engineToken, apiUrl }: { formData: FormData, engineToken: string, apiUrl: string }): Promise<StepFileUpsertResponse> {
+    const fetchWithRetry = fetchRetry(global.fetch)
+    const response = await fetchWithRetry(apiUrl + 'v1/step-files', {
+        method: 'POST',
+        headers: {
+            Authorization: 'Bearer ' + engineToken,
+        },
+        retryDelay: 3000,
+        retries: 3,
+        body: formData,
+    })
+
+    if (!response.ok) {
+        throw new FileStoreError({
+            status: response.status,
+            body: response.body,
+        })
+    }
+
+    return await response.json() as StepFileUpsertResponse
+}
+
+async function uploadFileContent({ url, data }: { url: string, data: Buffer }): Promise<void> {
+    const fetchWithRetry = fetchRetry(global.fetch)
+    const uploadResponse = await fetchWithRetry(url, {
+        method: 'PUT',
+        body: data,
+        headers: {
+            'Content-Type': 'application/octet-stream',
+        },
+        retries: 3,
+        retryDelay: 3000,
+    })
+
+    if (!uploadResponse.ok) {
+        throw new FileStoreError({
+            status: uploadResponse.status,
+            body: uploadResponse.body,
+        })
+    }
+}

activepieces-fork/packages/engine/src/lib/services/storage.service.ts

@@ -0,0 +1,190 @@
+import { URL } from 'node:url'
+import { Store, StoreScope } from '@activepieces/pieces-framework'
+import { DeleteStoreEntryRequest, ExecutionError, FetchError, FlowId, isNil, PutStoreEntryRequest, StorageError, StorageInvalidKeyError, StorageLimitError, STORE_KEY_MAX_LENGTH, STORE_VALUE_MAX_SIZE, StoreEntry } from '@activepieces/shared'
+import { StatusCodes } from 'http-status-codes'
+import sizeof from 'object-sizeof'
+import { utils } from '../utils'
+
+export const createStorageService = ({ engineToken, apiUrl }: CreateStorageServiceParams): StorageService => {
+    return {
+        async get(key: string): Promise<StoreEntry | null> {
+            const url = buildUrl(apiUrl, key)
+
+            const { data: storeEntry, error: storeEntryError } = await utils.tryCatchAndThrowOnEngineError((async () => {
+                const response = await fetch(url, {
+                    headers: {
+                        Authorization: `Bearer ${engineToken}`,
+                    },
+                })
+                if (!response.ok) {
+                    return handleResponseError({
+                        key,
+                        response,
+                    })
+                }
+
+                return response.json()
+            }))
+
+            if (storeEntryError) {
+                return handleFetchError({
+                    url,
+                    cause: storeEntryError,
+                })
+            }
+            return storeEntry
+        },
+
+        async put(request: PutStoreEntryRequest): Promise<StoreEntry | null> {
+            const url = buildUrl(apiUrl)
+
+            const { data: storeEntry, error: storeEntryError } = await utils.tryCatchAndThrowOnEngineError((async () => {
+                const sizeOfValue = sizeof(request.value)
+                if (sizeOfValue > STORE_VALUE_MAX_SIZE) {
+                    throw new StorageLimitError(request.key, STORE_VALUE_MAX_SIZE)
+                }
+                const response = await fetch(url, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        Authorization: `Bearer ${engineToken}`,
+                    },
+                    body: JSON.stringify(request),
+                })
+
+                if (!response.ok) {
+                    return handleResponseError({
+                        key: request.key,
+                        response,
+                    })
+                }
+
+                return response.json()
+            }))
+
+            if (storeEntryError) {
+                return handleFetchError({
+                    url,
+                    cause: storeEntryError,
+                })
+            }
+            return storeEntry
+        },
+
+        async delete(request: DeleteStoreEntryRequest): Promise<null> {
+            const url = buildUrl(apiUrl, request.key)
+
+            const { data: storeEntry, error: storeEntryError } = await utils.tryCatchAndThrowOnEngineError((async () => {
+                const response = await fetch(url, {
+                    method: 'DELETE',
+                    headers: {
+                        Authorization: `Bearer ${engineToken}`,
+                    },
+                })
+
+                if (!response.ok) {
+                    await handleResponseError({
+                        key: request.key,
+                        response,
+                    })
+                }
+
+                return null
+            }))
+            
+            if (storeEntryError) {
+                return handleFetchError({
+                    url,
+                    cause: storeEntryError,
+                })
+            }
+            return storeEntry
+        },
+    }
+}
+
+export function createContextStore({ apiUrl, prefix, flowId, engineToken }: { apiUrl: string, prefix: string, flowId: FlowId, engineToken: string }): Store {
+    return {
+        async put<T>(key: string, value: T, scope = StoreScope.FLOW): Promise<T> {
+            const modifiedKey = createKey(prefix, scope, flowId, key)
+            await createStorageService({ apiUrl, engineToken }).put({
+                key: modifiedKey,
+                value,
+            })
+            return value
+        },
+        async delete(key: string, scope = StoreScope.FLOW): Promise<void> {
+            const modifiedKey = createKey(prefix, scope, flowId, key)
+            await createStorageService({ apiUrl, engineToken }).delete({
+                key: modifiedKey,
+            })
+        },
+        async get<T>(key: string, scope = StoreScope.FLOW): Promise<T | null> {
+            const modifiedKey = createKey(prefix, scope, flowId, key)
+            const storeEntry = await createStorageService({ apiUrl, engineToken }).get(modifiedKey)
+            if (storeEntry === null) {
+                return null
+            }
+            return storeEntry.value as T
+        },
+    }
+}
+
+function createKey(prefix: string, scope: StoreScope, flowId: FlowId, key: string): string {
+    if (isNil(key) || typeof key !== 'string' || key.length === 0 || key.length > STORE_KEY_MAX_LENGTH) {
+        throw new StorageInvalidKeyError(key)
+    }
+    switch (scope) {
+        case StoreScope.PROJECT:
+            return prefix + key
+        case StoreScope.FLOW:
+            return prefix + 'flow_' + flowId + '/' + key
+    }
+}
+
+const buildUrl = (apiUrl: string, key?: string): URL => {
+    const url = new URL(`${apiUrl}v1/store-entries`)
+    if (key) {
+        url.searchParams.set('key', key)
+    }
+    return url
+}
+
+const handleResponseError = async ({ key, response }: HandleResponseErrorParams): Promise<null> => {
+    if (response.status === StatusCodes.NOT_FOUND.valueOf()) {
+        return null
+    }
+    if (response.status === StatusCodes.REQUEST_TOO_LONG) {
+        throw new StorageLimitError(key, STORE_VALUE_MAX_SIZE)
+    }
+    const cause = await response.text()
+    throw new StorageError(key, cause)
+}
+
+const handleFetchError = ({ url, cause }: HandleFetchErrorParams): never => {
+    if (cause instanceof ExecutionError) {
+        throw cause
+    }
+    throw new FetchError(url.toString(), cause)
+}
+
+type CreateStorageServiceParams = {
+    engineToken: string
+    apiUrl: string
+}
+
+type StorageService = {
+    get(key: string): Promise<StoreEntry | null>
+    put(request: PutStoreEntryRequest): Promise<StoreEntry | null>
+    delete(request: DeleteStoreEntryRequest): Promise<null>
+}
+
+type HandleResponseErrorParams = {
+    key: string
+    response: Response
+}
+
+type HandleFetchErrorParams = {
+    url: URL
+    cause: unknown
+}

activepieces-fork/packages/engine/src/lib/tools/index.ts

@@ -0,0 +1,260 @@
+import { Action, DropdownOption, ExecutePropsResult, PieceProperty, PropertyType } from '@activepieces/pieces-framework'
+import { AgentTool, AgentToolType, ExecuteToolOperation, ExecuteToolResponse, ExecutionToolStatus, FlowActionType, isNil, PieceAction, PropertyExecutionType, StepOutputStatus } from '@activepieces/shared'
+import { generateObject, LanguageModel, ToolSet } from 'ai'
+import { z } from 'zod/v4'
+import { EngineConstants } from '../handler/context/engine-constants'
+import { FlowExecutorContext } from '../handler/context/flow-execution-context'
+import { flowExecutor } from '../handler/flow-executor'
+import { pieceHelper } from '../helper/piece-helper'
+import { pieceLoader } from '../helper/piece-loader'
+import { tsort } from './tsort'
+
+export const agentTools = {
+    async tools({ engineConstants, tools, model }: ConstructToolParams): Promise<ToolSet> {
+        const piecesTools = await Promise.all(tools
+            .filter((tool) => tool.type === AgentToolType.PIECE)
+            .map(async (tool) => {
+                const { pieceAction } = await pieceLoader.getPieceAndActionOrThrow({
+                    pieceName: tool.pieceMetadata.pieceName,
+                    pieceVersion: tool.pieceMetadata.pieceVersion,
+                    actionName: tool.pieceMetadata.actionName,
+                    devPieces: EngineConstants.DEV_PIECES,
+                })
+                return {
+                    name: tool.toolName,
+                    description: pieceAction.description,
+                    inputSchema: z.object({
+                        instruction: z.string().describe('The instruction to the tool'),
+                    }),
+                    execute: async ({ instruction }: { instruction: string }) =>
+                        execute({
+                            ...engineConstants,
+                            instruction,
+                            pieceName: tool.pieceMetadata.pieceName,
+                            pieceVersion: tool.pieceMetadata.pieceVersion,
+                            actionName: tool.pieceMetadata.actionName,
+                            predefinedInput: tool.pieceMetadata.predefinedInput,
+                            model,
+                        }),
+                }
+            }))
+  
+        return {
+            ...Object.fromEntries(piecesTools.map((tool) => [tool.name, tool])),
+        }
+    },
+}
+
+async function resolveProperties(depthToPropertyMap: Record<number, string[]>, instruction: string, action: Action, model: LanguageModel, operation: ExecuteToolOperation): Promise<Record<string, unknown>> {
+    let result: Record<string, unknown> = operation.predefinedInput
+    for (const [_, properties] of Object.entries(depthToPropertyMap)) {
+        const propertyToFill: Record<string, z.ZodTypeAny> = {}
+        const propertyPrompts: string[] = []
+        for (const property of properties) {
+            const propertyFromAction = action.props[property]
+            const propertyType = propertyFromAction.type
+            const skip = [PropertyType.BASIC_AUTH, PropertyType.OAUTH2, PropertyType.CUSTOM_AUTH, PropertyType.CUSTOM, PropertyType.MARKDOWN]
+            if (skip.includes(propertyType) || property in operation.predefinedInput) {
+                continue
+            }
+            const propertyPrompt = await buildPromptForProperty(property, propertyFromAction, operation, result)
+            if (!isNil(propertyPrompt)) {
+                propertyPrompts.push(propertyPrompt)
+            }
+            const propertySchema = await propertyToSchema(property, propertyFromAction, operation, result)
+            propertyToFill[property] = propertyFromAction.required ? propertySchema : propertySchema.nullish()
+        }
+        const schemaObject = z.object(propertyToFill) as z.ZodTypeAny
+        const extractionPrompt = constructExtractionPrompt(instruction, propertyToFill, propertyPrompts)
+
+        const { object } = await generateObject({
+            model,
+            schema: schemaObject,
+            prompt: extractionPrompt,
+        })
+        result = {
+            ...result,
+            ...(object as Record<string, unknown>),
+        }
+    }
+    return result
+}
+
+
+async function execute(operation: ExecuteToolOperationWithModel): Promise<ExecuteToolResponse> {
+    const { pieceAction } = await pieceLoader.getPieceAndActionOrThrow({
+        pieceName: operation.pieceName,
+        pieceVersion: operation.pieceVersion,
+        actionName: operation.actionName,
+        devPieces: EngineConstants.DEV_PIECES,
+    })
+    const depthToPropertyMap = tsort.sortPropertiesByDependencies(pieceAction.props)
+    const resolvedInput = await resolveProperties(depthToPropertyMap, operation.instruction, pieceAction, operation.model, operation)
+    const step: PieceAction = {
+        name: operation.actionName,
+        displayName: operation.actionName,
+        type: FlowActionType.PIECE,
+        settings: {
+            input: resolvedInput,
+            actionName: operation.actionName,
+            pieceName: operation.pieceName,
+            pieceVersion: operation.pieceVersion,
+            propertySettings: Object.fromEntries(Object.entries(resolvedInput).map(([key]) => [key, {
+                type: PropertyExecutionType.MANUAL,
+                schema: undefined,
+            }])),
+        },
+        valid: true,
+    }
+
+    const output = await flowExecutor.getExecutorForAction(step.type).handle({
+        action: step,
+        executionState: FlowExecutorContext.empty(),
+        constants: EngineConstants.fromExecuteActionInput(operation),
+    })
+
+    const { output: stepOutput, errorMessage, status } = output.steps[operation.actionName]
+
+    return {
+        status: status === StepOutputStatus.FAILED ? ExecutionToolStatus.FAILED : ExecutionToolStatus.SUCCESS,
+        output: stepOutput,
+        resolvedInput: {
+            ...resolvedInput,
+            auth: 'Redacted',
+        },
+        errorMessage,
+    }
+}
+
+
+const constructExtractionPrompt = (instruction: string, propertyToFill: Record<string, z.ZodTypeAny>, propertyPrompts: string[]): string => {
+    const propertyNames = Object.keys(propertyToFill).join('", "')
+    return `
+You are an expert at understanding API schemas and filling out properties based on user instructions.
+
+TASK: Fill out the properties "${propertyNames}" based on the user's instructions.
+
+USER INSTRUCTIONS:
+${instruction}
+
+${propertyPrompts.join('\n')}
+
+IMPORTANT:
+- For dropdown, multi-select dropdown, and static dropdown properties, YOU MUST SELECT VALUES FROM THE PROVIDED OPTIONS ARRAY ONLY.
+- For array properties, YOU MUST SELECT VALUES FROM THE PROVIDED OPTIONS ARRAY ONLY.
+- For dynamic properties, YOU MUST SELECT VALUES FROM THE PROVIDED OPTIONS ARRAY ONLY.
+- THE OPTIONS ARRAY WILL BE [{ label: string, value: string | object }]. YOU MUST SELECT THE value FIELD FROM THE OPTION OBJECT.
+- For DATE_TIME properties, return date strings in ISO format (YYYY-MM-DDTHH:mm:ss.sssZ)
+- Use actual values from the user instructions to determine the correct value for each property, either as a hint for selecting options from dropdowns or to fill in the property if possible.
+- Must include all required properties, even if the user does not provide a value. If a required field is missing, look up the correct value or provide a reasonable default—otherwise, the task may fail.
+- IMPORTANT: If a property is not required and you do not have any information to fill it, you MUST skip it.
+`
+}
+
+type ExecuteToolOperationWithModel = ExecuteToolOperation & {
+    model: LanguageModel
+}
+
+
+async function propertyToSchema(propertyName: string, property: PieceProperty, operation: ExecuteToolOperation, resolvedInput: Record<string, unknown>): Promise<z.ZodTypeAny> {
+    let schema: z.ZodTypeAny
+    switch (property.type) {
+        case PropertyType.SHORT_TEXT:
+        case PropertyType.LONG_TEXT:
+        case PropertyType.MARKDOWN:
+        case PropertyType.DATE_TIME:
+        case PropertyType.FILE:
+        case PropertyType.COLOR:
+            schema = z.string()
+            break
+        case PropertyType.DROPDOWN:
+        case PropertyType.STATIC_DROPDOWN: {
+            schema = z.union([z.string(), z.number(), z.record(z.string(), z.unknown())])
+            break
+        }
+        case PropertyType.MULTI_SELECT_DROPDOWN:
+        case PropertyType.STATIC_MULTI_SELECT_DROPDOWN: {
+            schema = z.union([z.array(z.string()), z.array(z.record(z.string(), z.unknown()))])
+            break
+        }
+        case PropertyType.NUMBER:
+            schema = z.number()
+            break
+        case PropertyType.ARRAY:
+            return z.array(z.unknown())
+        case PropertyType.OBJECT:
+            schema = z.record(z.string(), z.unknown())
+            break
+        case PropertyType.JSON:
+            schema = z.record(z.string(), z.unknown())
+            break
+        case PropertyType.DYNAMIC: {
+            schema = await buildDynamicSchema(propertyName, operation, resolvedInput)
+            break
+        }
+        case PropertyType.CHECKBOX:
+            schema = z.boolean()
+            break
+        case PropertyType.CUSTOM:
+            schema = z.string()
+            break
+        case PropertyType.OAUTH2:
+        case PropertyType.BASIC_AUTH:
+        case PropertyType.CUSTOM_AUTH:
+        case PropertyType.SECRET_TEXT:
+            throw new Error(`Unsupported property type: ${property.type}`)
+    }
+    if (property.defaultValue) {
+        schema = schema.default(property.defaultValue)
+    }
+    if (property.description) {
+        schema = schema.describe(property.description)
+    }
+    return property.required ? schema : schema.nullish()
+}
+
+async function buildDynamicSchema(propertyName: string, operation: ExecuteToolOperation, resolvedInput: Record<string, unknown>): Promise<z.ZodTypeAny> {
+    const response = await pieceHelper.executeProps({
+        ...operation,
+        propertyName,
+        actionOrTriggerName: operation.actionName,
+        input: resolvedInput,
+        sampleData: {},
+        searchValue: undefined,
+    }) as unknown as ExecutePropsResult<PropertyType.DYNAMIC>
+    const dynamicProperties = response.options
+    const dynamicSchema: Record<string, z.ZodTypeAny> = {}
+    for (const [key, value] of Object.entries(dynamicProperties)) {
+        dynamicSchema[key] = await propertyToSchema(key, value, operation, resolvedInput)
+    }
+    return z.object(dynamicSchema)
+}
+
+
+async function buildPromptForProperty(propertyName: string, property: PieceProperty, operation: ExecuteToolOperation, input: Record<string, unknown>): Promise<string | null> {
+    if (property.type === PropertyType.DROPDOWN || property.type === PropertyType.MULTI_SELECT_DROPDOWN) {
+        const options = await loadOptions(propertyName, operation, input)
+        return `The options for the property "${propertyName}" are: ${JSON.stringify(options)}`
+    }
+    return null
+}
+
+async function loadOptions(propertyName: string, operation: ExecuteToolOperation, input: Record<string, unknown>): Promise<DropdownOption<unknown>[]> {
+    const response = await pieceHelper.executeProps({
+        ...operation,
+        propertyName,
+        actionOrTriggerName: operation.actionName,
+        input,
+        sampleData: {},
+        searchValue: undefined,
+    }) as unknown as ExecutePropsResult<PropertyType.DROPDOWN | PropertyType.MULTI_SELECT_DROPDOWN>
+    const options = response.options
+    return options.options
+}
+
+
+type ConstructToolParams = {
+    engineConstants: EngineConstants
+    tools: AgentTool[]
+    model: LanguageModel
+}

activepieces-fork/packages/engine/src/lib/tools/tsort.ts

@@ -0,0 +1,56 @@
+import { PiecePropertyMap } from '@activepieces/pieces-framework'
+
+export const tsort = {
+    sortPropertiesByDependencies(properties: PiecePropertyMap): Record<number, string[]> {
+        const inDegree: Record<string, number> = {}
+        const graph: Record<string, string[]> = {}
+        const depth: Record<string, number> = {}
+
+        Object.entries(properties).forEach(([key, property]) => {
+            const hasRefreshers = 'refreshers' in property && property.refreshers && Array.isArray(property.refreshers) && property.refreshers.length > 0
+            if (hasRefreshers) {
+                for (const refresher of property.refreshers) {
+                    if (typeof properties[refresher] === 'undefined' || properties[refresher] === null) {
+                        continue
+                    }
+                    inDegree[key] = (inDegree[key] || 0) + 1
+                    graph[refresher] = graph[refresher] ?? []
+                    graph[refresher].push(key)
+                }
+            }
+            inDegree[key] = inDegree[key] ?? 0
+            graph[key] = graph[key] ?? []
+        })
+
+        // Topological sort
+        const order: string[] = []
+        const queue = Object.entries(inDegree)
+            .filter(([, degree]) => degree === 0)
+            .map(([name]) => name)
+
+        queue.forEach(property => depth[property] = 0)
+
+        while (queue.length > 0) {
+            const current = queue.shift()!
+            order.push(current)
+
+            const neighbors = graph[current] || []
+            neighbors.forEach(neighbor => {
+                inDegree[neighbor]--
+                if (inDegree[neighbor] === 0) {
+                    queue.push(neighbor)
+                    depth[neighbor] = depth[current] + 1
+                }
+            })
+        }
+
+        const depthToPropertyMap: Record<number, string[]> = {}
+        for (const [property, depthValue] of Object.entries(depth)) {
+            depthToPropertyMap[depthValue] = depthToPropertyMap[depthValue] ?? []
+            depthToPropertyMap[depthValue].push(property)
+        }
+
+        return depthToPropertyMap
+    },
+}
+

activepieces-fork/packages/engine/src/lib/utils.ts

@@ -0,0 +1,106 @@
+import fs from 'fs/promises'
+import { inspect } from 'node:util'
+import path from 'path'
+import { ConnectionsManager, ContextVersion, PauseHookParams, RespondHookParams, StopHookParams } from '@activepieces/pieces-framework'
+import { ExecutionError, ExecutionErrorType, Result, tryCatch } from '@activepieces/shared'
+import { createConnectionService } from './services/connections.service'
+
+export type FileEntry = {
+    name: string
+    path: string
+}
+
+export const utils = {
+    async tryCatchAndThrowOnEngineError<T>(fn: () => Promise<T>): Promise<Result<T, ExecutionError>> {
+        const result = await tryCatch<T, ExecutionError>(fn)
+        if (isEngineError(result.error)) {
+            throw result.error
+        }
+        return result
+    },
+    async walk(dirPath: string): Promise<FileEntry[]> {
+        const entries: FileEntry[] = []
+
+        async function walkRecursive(currentPath: string) {
+            const items = await fs.readdir(currentPath, { withFileTypes: true })
+
+            for (const item of items) {
+                const fullPath = path.join(currentPath, item.name)
+                const absolutePath = path.resolve(fullPath)
+
+                entries.push({
+                    name: item.name,
+                    path: absolutePath,
+                })
+
+                if (item.isDirectory()) {
+                    await walkRecursive(fullPath)
+                }
+            }
+        }
+
+        await walkRecursive(dirPath)
+        return entries
+    },
+    formatExecutionError(value: ExecutionError): string {
+        try {
+            return JSON.stringify({
+                ...value,
+                ...JSON.parse(value.message),
+            }, null, 2)
+        }
+        catch (e) {
+            return inspect(value)
+        }
+    },
+    formatError(value: Error): string {
+        try {
+            return JSON.stringify(JSON.parse(value.message), null, 2)
+        }
+        catch (e) {
+            return inspect(value)
+        }
+    },
+    async folderExists(filePath: string): Promise<boolean> {
+        try {
+            await fs.access(filePath)
+            return true
+        }
+        catch {
+            return false
+        }
+    },
+    createConnectionManager(params: CreateConnectionManagerParams): ConnectionsManager {
+        return {
+            get: async (key: string) => {
+                const connection = await createConnectionService({ projectId: params.projectId, engineToken: params.engineToken, apiUrl: params.apiUrl, contextVersion: params.contextVersion }).obtain(key)
+                if (params.target === 'actions') {
+                    params.hookResponse.tags.push(`connection:${key}`)
+                }
+                return connection
+            },
+        }
+    },
+}
+
+function isEngineError(error: unknown): error is ExecutionError {
+    return error instanceof ExecutionError && error.type === ExecutionErrorType.ENGINE
+}
+
+export type HookResponse = {
+    type: 'paused'
+    tags: string[]
+    response: PauseHookParams
+} | {
+    type: 'stopped'
+    tags: string[]
+    response: StopHookParams
+} | {
+    type: 'respond'
+    tags: string[]
+    response: RespondHookParams
+} | {
+    type: 'none'
+    tags: string[]
+}
+type CreateConnectionManagerParams = { projectId: string, engineToken: string, apiUrl: string, target: 'triggers' | 'properties', contextVersion: ContextVersion | undefined } | { projectId: string, engineToken: string, apiUrl: string, target: 'actions', hookResponse: HookResponse, contextVersion: ContextVersion | undefined }

activepieces-fork/packages/engine/src/lib/variables/processors/array-zipper.ts

@@ -0,0 +1,27 @@
+import { isObject } from '@activepieces/shared'
+import { ProcessorFn } from './types'
+
+function getLongestArrayLengthInObject(props: Record<string, unknown>): number {
+    return Math.max(
+        ...Object.values(props).map(value =>
+            Array.isArray(value) ? value.length : 1,
+        ),
+    )
+}
+
+function constructResultForIndex(props: Record<string, unknown>, index: number): Record<string, unknown> {
+    return Object.entries(props).reduce((result, [key, value]) => {
+        result[key] = Array.isArray(value) ? value[index] : value
+        return result
+    }, {} as Record<string, unknown>)
+}
+
+export const arrayZipperProcessor: ProcessorFn = (_property, value) => {
+    if (Array.isArray(value) || !isObject(value)) {
+        return value
+    }
+  
+    return Array.from({ length: getLongestArrayLengthInObject(value) },
+        (_, index) => constructResultForIndex(value, index),
+    )
+}

activepieces-fork/packages/engine/src/lib/variables/processors/date-time.ts

@@ -0,0 +1,18 @@
+import dayjs from 'dayjs'
+import timezone from 'dayjs/plugin/timezone'
+import utc from 'dayjs/plugin/utc'
+import { ProcessorFn } from './types'
+
+export const dateTimeProcessor: ProcessorFn = (_property, value) => {
+    dayjs.extend(utc)
+    dayjs.extend(timezone)
+    const dateTimeString = value
+    try {
+        if (!dateTimeString) throw Error('Undefined input')
+        return dayjs.tz(dateTimeString, 'UTC').toISOString()
+    }
+    catch (error) {
+        console.error(error)
+        return undefined
+    }
+}

activepieces-fork/packages/engine/src/lib/variables/processors/file.ts

@@ -0,0 +1,89 @@
+import { ApFile } from '@activepieces/pieces-framework'
+import { isNil, isString } from '@activepieces/shared'
+import axios from 'axios'
+import isBase64 from 'is-base64'
+import mime from 'mime-types'
+import { ProcessorFn } from './types'
+
+export const fileProcessor: ProcessorFn = async (_property, urlOrBase64) => {
+    if (isNil(urlOrBase64) || !isString(urlOrBase64)) {
+        return null
+    }
+    try {
+        const file = handleBase64File(urlOrBase64)
+        if (!isNil(file)) {
+            return file
+        }
+        return await handleUrlFile(urlOrBase64)
+    }
+    catch (e) {
+        console.error(e)
+        return null
+    }
+}
+
+function handleBase64File(propertyValue: string): ApFile | null {
+    if (!isBase64(propertyValue, { allowMime: true })) {
+        return null
+    }
+    const matches = propertyValue.match(/^data:([A-Za-z-+/]+);base64,(.+)$/) // example match: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAQMAAAAlPW0iAAAABlBMVEUAAAD///+l2Z/dAAAAM0lEQVR4nGP4/5/h/1+G/58ZDrAz3D/McH8yw83NDDeNGe4Ug9C9zwz3gVLMDA/A6P9/AFGGFyjOXZtQAAAAAElFTkSuQmCC
+    if (!matches || matches?.length !== 3) {
+        return null
+    }
+    const base64 = matches[2]
+    const extension = mime.extension(matches[1]) || 'bin'
+    return new ApFile(
+        `unknown.${extension}`,
+        Buffer.from(base64, 'base64'),
+        extension,
+    )
+}
+
+async function handleUrlFile(path: string): Promise<ApFile | null> {
+    const fileResponse = await axios.get(path, {
+        responseType: 'arraybuffer',
+    })
+
+
+    const filename = getFileName(path, fileResponse.headers['content-disposition'], fileResponse.headers['content-type']) ?? 'unknown'
+    const extension = filename.split('.').length > 1 ? filename.split('.').pop() : undefined
+
+    return new ApFile(
+        filename,
+        Buffer.from(fileResponse.data, 'binary'),
+        extension,
+    )
+}
+
+
+function getFileName(path: string, disposition: string | null, mimeType: string | undefined): string | null {
+    const url = new URL(path)
+    if (isNil(disposition)) {
+        const fileNameFromUrl = url.pathname.includes('/') && url.pathname.split('/').pop()?.includes('.') ? url.pathname.split('/').pop() : null
+        if (!isNil(fileNameFromUrl)) {
+            return fileNameFromUrl
+        }
+        const resolvedExtension = mimeType ? mime.extension(mimeType) : null
+        return `unknown.${resolvedExtension ?? 'bin'}`
+    }
+    const utf8FilenameRegex = /filename\*=UTF-8''([\w%\-.]+)(?:; ?|$)/i
+    if (utf8FilenameRegex.test(disposition)) {
+        const result = utf8FilenameRegex.exec(disposition)
+        if (result && result.length > 1) {
+            return decodeURIComponent(result[1])
+        }
+    }
+    // prevent ReDos attacks by anchoring the ascii regex to string start and
+    // slicing off everything before 'filename='
+    const filenameStart = disposition.toLowerCase().indexOf('filename=')
+    const asciiFilenameRegex = /^filename=(["']?)(.*?[^\\])\1(?:; ?|$)/i
+
+    if (filenameStart >= 0) {
+        const partialDisposition = disposition.slice(filenameStart)
+        const matches = asciiFilenameRegex.exec(partialDisposition)
+        if (matches != null && matches[2]) {
+            return matches[2]
+        }
+    }
+    return null
+}

activepieces-fork/packages/engine/src/lib/variables/processors/index.ts

@@ -0,0 +1,19 @@
+import { PropertyType } from '@activepieces/pieces-framework'
+import { dateTimeProcessor } from './date-time'
+import { fileProcessor } from './file'
+import { jsonProcessor } from './json'
+import { numberProcessor } from './number'
+import { objectProcessor } from './object'
+import { textProcessor } from './text'
+import { ProcessorFn } from './types'
+
+export const processors: Partial<Record<PropertyType, ProcessorFn>> = {
+    JSON: jsonProcessor,
+    OBJECT: objectProcessor,
+    NUMBER: numberProcessor,
+    LONG_TEXT: textProcessor,
+    SHORT_TEXT: textProcessor,
+    SECRET_TEXT: textProcessor,
+    DATE_TIME: dateTimeProcessor,
+    FILE: fileProcessor,
+}

activepieces-fork/packages/engine/src/lib/variables/processors/json.ts

@@ -0,0 +1,18 @@
+import { isNil } from '@activepieces/shared'
+import { ProcessorFn } from './types'
+
+export const jsonProcessor: ProcessorFn = (_property, value) => {
+    if (isNil(value)) {
+        return value
+    }
+    try {
+        if (typeof value === 'object') {
+            return value
+        }
+        return JSON.parse(value)
+    }
+    catch (error) {
+        console.error(error)
+        return undefined
+    }
+}

activepieces-fork/packages/engine/src/lib/variables/processors/number.ts

@@ -0,0 +1,12 @@
+import { isNil } from '@activepieces/shared'
+import { ProcessorFn } from './types'
+
+export const numberProcessor: ProcessorFn = (_property, value) => {
+    if (isNil(value)) {
+        return value
+    }
+    if (value === '') {
+        return undefined
+    }
+    return Number(value)
+}

activepieces-fork/packages/engine/src/lib/variables/processors/object.ts

@@ -0,0 +1,20 @@
+import { isNil } from '@activepieces/shared'
+import { ProcessorFn } from './types'
+
+export const objectProcessor: ProcessorFn = (_property, value) => {
+    if (isNil(value)) {
+        return value
+    }
+    if (typeof value === 'string') {
+        try {
+            return JSON.parse(value)
+        }
+        catch (e) {
+            return undefined
+        }
+    }
+    if (typeof value === 'object' && !Array.isArray(value)) {
+        return value
+    }
+    return undefined
+}

activepieces-fork/packages/engine/src/lib/variables/processors/text.ts

@@ -0,0 +1,17 @@
+import { isNil } from '@activepieces/shared'
+import { ProcessorFn } from './types'
+
+export const textProcessor: ProcessorFn = (property, value) => {
+    if (isNil(value)) {
+        return value
+    }
+    if (typeof value === 'object') {
+        return JSON.stringify(value)
+    }
+
+    const result = value.toString()
+    if (result.length === 0 && !property.required) {
+        return undefined
+    }
+    return result
+}

activepieces-fork/packages/engine/src/lib/variables/processors/types.ts

@@ -0,0 +1,8 @@
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { PieceProperty } from '@activepieces/pieces-framework'
+
+export type ProcessorFn<INPUT = any, OUTPUT = any> = (
+    property: PieceProperty,
+    value: INPUT,
+) => OUTPUT

activepieces-fork/packages/engine/src/lib/variables/props-processor.ts

@@ -0,0 +1,197 @@
+import { getAuthPropertyForValue, InputPropertyMap, PieceAuthProperty, PieceProperty, PiecePropertyMap, PropertyType, StaticPropsValue } from '@activepieces/pieces-framework'
+import { AppConnectionValue, AUTHENTICATION_PROPERTY_NAME, isNil, isObject, PropertySettings } from '@activepieces/shared'
+import { z } from 'zod'
+import { processors } from './processors'
+import { arrayZipperProcessor } from './processors/array-zipper'
+
+type PropsValidationError = {
+    [key: string]: string[] | PropsValidationError | PropsValidationError[]
+}
+
+export const propsProcessor = {
+    applyProcessorsAndValidators: async (
+        resolvedInput: StaticPropsValue<PiecePropertyMap>,
+        props: InputPropertyMap,
+        auth: PieceAuthProperty | PieceAuthProperty[] | undefined,
+        requireAuth: boolean,
+        propertySettings: Record<string, PropertySettings>,
+    ): Promise<{ processedInput: StaticPropsValue<PiecePropertyMap>, errors: PropsValidationError }> => {
+        let dynamaicPropertiesSchema: Record<string, InputPropertyMap> | undefined = undefined
+        if (Object.keys(propertySettings).length > 0) {
+            dynamaicPropertiesSchema = Object.fromEntries(Object.entries(propertySettings).map(([key, propertySetting]) => [key, propertySetting.schema]))
+        }
+        const processedInput = { ...resolvedInput }
+        const errors: PropsValidationError = {}
+        const authValue: AppConnectionValue | undefined = resolvedInput[AUTHENTICATION_PROPERTY_NAME]
+        if (authValue && requireAuth) {
+            const authPropsToProcess = getAuthPropsToProcess(authValue, auth)
+            if (authPropsToProcess) {
+                const { processedInput: authProcessedInput, errors: authErrors } = await propsProcessor.applyProcessorsAndValidators(
+                    resolvedInput[AUTHENTICATION_PROPERTY_NAME],
+                    authPropsToProcess,
+                    undefined,
+                    false,
+                    {},
+                )
+                processedInput[AUTHENTICATION_PROPERTY_NAME] = authProcessedInput
+                if (Object.keys(authErrors).length > 0) {
+                    errors[AUTHENTICATION_PROPERTY_NAME] = authErrors
+                }
+            }
+        }
+        for (const [key, value] of Object.entries(resolvedInput)) {
+            const property = props[key]
+            if (isNil(property)) {
+                continue
+            }
+            if (property.type === PropertyType.DYNAMIC && !isNil(dynamaicPropertiesSchema?.[key])) {
+                const { processedInput: itemProcessedInput, errors: itemErrors } = await propsProcessor.applyProcessorsAndValidators(
+                    value,
+                    dynamaicPropertiesSchema[key],
+                    undefined,
+                    false,
+                    {},
+                )
+                processedInput[key] = itemProcessedInput
+                if (Object.keys(itemErrors).length > 0) {
+                    errors[key] = itemErrors
+                }
+            }
+            if (property.type === PropertyType.ARRAY && property.properties) {
+                const arrayOfObjects = arrayZipperProcessor(property, value) ?? []
+                const processedArray = []
+                const processedErrors = []
+                for (const item of arrayOfObjects) {
+                    const { processedInput: itemProcessedInput, errors: itemErrors } = await propsProcessor.applyProcessorsAndValidators(
+                        item,
+                        property.properties,
+                        undefined,
+                        false,
+                        {},
+                    )
+                    processedArray.push(itemProcessedInput)
+                    processedErrors.push(itemErrors)
+                }
+                processedInput[key] = processedArray
+                const isThereErrors = processedErrors.some(error => Object.keys(error).length > 0)
+                if (isThereErrors) {
+                    errors[key] = {
+                        properties: processedErrors,
+                    }
+                }
+            }
+            const processor = processors[property.type]
+            if (processor) {
+                processedInput[key] = await processor(property, processedInput[key])
+            }
+
+            const shouldValidate = key !== AUTHENTICATION_PROPERTY_NAME && property.type !== PropertyType.MARKDOWN
+            if (!shouldValidate) {
+                continue
+            }
+        }
+
+        for (const [key, value] of Object.entries(processedInput)) {
+            const property = props[key]
+            if (isNil(property)) {
+                continue
+            }
+
+            const validationErrors = validateProperty(property, value, resolvedInput[key])
+            if (validationErrors.length > 0) {
+                errors[key] = validationErrors
+            }
+        }
+
+        return { processedInput, errors }
+    },
+}
+
+const validateProperty = (property: PieceProperty, value: unknown, originalValue: unknown): string[] => {
+    let schema
+    switch (property.type) {
+        case PropertyType.SHORT_TEXT:
+        case PropertyType.LONG_TEXT:
+            schema = z.string({
+                error: `Expected string, received: ${originalValue}`,
+            })
+            break
+        case PropertyType.NUMBER:
+            schema = z.number({
+                error: `Expected number, received: ${originalValue}`,
+            })
+            break
+        case PropertyType.CHECKBOX:
+            schema = z.boolean({
+                error: `Expected boolean, received: ${originalValue}`,
+            })
+            break
+        case PropertyType.DATE_TIME:
+            schema = z.string({
+                error: `Invalid datetime format. Expected ISO format (e.g. 2024-03-14T12:00:00.000Z), received: ${originalValue}`,
+            })
+            break
+        case PropertyType.ARRAY:
+            schema = z.array(z.any(), {
+                error: `Expected array, received: ${originalValue}`,
+            })
+            break
+        case PropertyType.OBJECT:
+            schema = z.record(z.any(), z.any(), {
+                error: `Expected object, received: ${originalValue}`,
+            })
+            break
+        case PropertyType.JSON:
+            schema = z.any().refine(
+                (val) => isObject(val) || Array.isArray(val),
+                {
+                    message: `Expected JSON, received: ${originalValue}`,
+                },
+            )
+            break
+        case PropertyType.FILE: {
+            schema = z.any().refine(
+                (val) => isObject(val),
+                {
+                    message: `Expected file url or base64 with mimeType, received: ${originalValue}`,
+                },
+            )
+            break
+        }
+        default:
+            schema = z.any()
+    }
+    let finalSchema
+    if (property.required) {
+        finalSchema = schema
+    }
+    else {
+        finalSchema = schema.nullable().optional()
+    }
+
+    try {
+        finalSchema.parse(value)
+        return []
+    }
+    catch (err) {
+        if (err instanceof z.ZodError) {
+            return err.issues.map(e => e.message)
+        }
+        return []
+    }
+}
+
+function getAuthPropsToProcess(authValue: AppConnectionValue, auth: PieceAuthProperty | PieceAuthProperty[] | undefined):  | null {
+    if (isNil(auth)) {
+        return null
+    }
+    const usedAuthProperty = getAuthPropertyForValue({
+        authValueType: authValue.type,
+        pieceAuth: auth,
+    })
+    const doesAuthHaveProps = usedAuthProperty?.type === PropertyType.CUSTOM_AUTH || usedAuthProperty?.type === PropertyType.OAUTH2
+    if (doesAuthHaveProps && !isNil(usedAuthProperty?.props)) {
+        return usedAuthProperty.props
+    }
+    return null
+}

activepieces-fork/packages/engine/src/lib/variables/props-resolver.ts

@@ -0,0 +1,254 @@
+import { ContextVersion } from '@activepieces/pieces-framework'
+import { applyFunctionToValues, isNil, isString } from '@activepieces/shared'
+import replaceAsync from 'string-replace-async'
+import { initCodeSandbox } from '../core/code/code-sandbox'
+import { FlowExecutorContext } from '../handler/context/flow-execution-context'
+import { createConnectionService } from '../services/connections.service'
+import { utils } from '../utils'
+
+const VARIABLE_PATTERN = /\{\{(.*?)\}\}/g
+const CONNECTIONS = 'connections'
+const FLATTEN_NESTED_KEYS_PATTERN = /\{\{\s*flattenNestedKeys(.*?)\}\}/g
+
+
+export const createPropsResolver = ({ engineToken, projectId, apiUrl, contextVersion }: PropsResolverParams) => {
+    return {
+        resolve: async <T = unknown>(params: ResolveInputParams): Promise<ResolveResult<T>> => {
+            const { unresolvedInput, executionState } = params
+            if (isNil(unresolvedInput)) {
+                return {
+                    resolvedInput: unresolvedInput as T,
+                    censoredInput: unresolvedInput,
+                }
+            }
+            const currentState = executionState.currentState()
+            const resolveOptions = {
+                engineToken,
+                projectId,
+                apiUrl,
+                currentState,
+            }
+            const resolvedInput = await applyFunctionToValues<T>(
+                unresolvedInput,
+                (token) => resolveInputAsync({
+                    ...resolveOptions,
+                    input: token,
+                    censoredInput: false,
+                    contextVersion,
+                }))
+            const censoredInput = await applyFunctionToValues<T>(
+                unresolvedInput,
+                (token) => resolveInputAsync({
+                    ...resolveOptions,
+                    input: token,
+                    censoredInput: true,
+                    contextVersion,
+                }))
+            return {
+                resolvedInput,
+                censoredInput,
+            }
+        },
+    }
+}
+
+const mergeFlattenedKeysArraysIntoOneArray = async (token: string, partsThatNeedResolving: string[],
+    resolveOptions: Pick<ResolveInputInternalParams, 'engineToken' | 'projectId' | 'apiUrl' | 'currentState' | 'censoredInput'>,
+    contextVersion: ContextVersion | undefined,
+) => {
+    const resolvedValues: Record<string, unknown> = {}
+    let longestResultLength = 0
+    for (const tokenPart of partsThatNeedResolving) {
+        const variableName = tokenPart.substring(2, tokenPart.length - 2)
+        resolvedValues[tokenPart] = await resolveSingleToken({
+            ...resolveOptions,
+            variableName,
+            contextVersion,
+        })
+        if (Array.isArray(resolvedValues[tokenPart])) {
+            longestResultLength = Math.max(longestResultLength, resolvedValues[tokenPart].length)
+        }
+    }
+    const result = new Array(longestResultLength).fill(null).map((_, index) => {
+        return Object.entries(resolvedValues).reduce((acc, [tokenPart, value]) => {
+            const valueToUse = (Array.isArray(value) ? value[index] : value) ?? ''
+            acc = acc.replace(tokenPart, isString(valueToUse) ? valueToUse : JSON.stringify(valueToUse))
+            return acc
+        }, token)
+    })
+    return result
+}
+
+export type PropsResolver = ReturnType<typeof createPropsResolver>
+/** 
+ * input: `Hello {{firstName}} {{lastName}}`
+ * tokenThatNeedResolving: [`{{firstName}}`, `{{lastName}}`]
+ */
+async function resolveInputAsync(params: ResolveInputInternalParams): Promise<unknown> {
+    const { input, currentState, engineToken, projectId, apiUrl, censoredInput } = params
+    const tokensThatNeedResolving = input.match(VARIABLE_PATTERN)
+    const inputContainsOnlyOneTokenToResolve = tokensThatNeedResolving !== null && tokensThatNeedResolving.length === 1 && tokensThatNeedResolving[0] === input
+    const resolveOptions = {
+        engineToken,
+        projectId,
+        apiUrl,
+        currentState,
+        censoredInput,
+    }
+
+    if (inputContainsOnlyOneTokenToResolve) {
+        const trimmedInput = input.trim()
+        const variableName = trimmedInput.substring(2, trimmedInput.length - 2)
+        return resolveSingleToken({
+            ...resolveOptions,
+            variableName,
+            contextVersion: params.contextVersion,
+        })
+    }
+    const inputIncludesFlattenNestedKeysTokens = input.match(FLATTEN_NESTED_KEYS_PATTERN)
+    if (!isNil(inputIncludesFlattenNestedKeysTokens) && !isNil(tokensThatNeedResolving)) {
+        return mergeFlattenedKeysArraysIntoOneArray(input, tokensThatNeedResolving, resolveOptions, params.contextVersion)
+    }
+
+    return replaceAsync(input, VARIABLE_PATTERN, async (_fullMatch, variableName) => {
+        const result = await resolveSingleToken({
+            ...resolveOptions,
+            variableName,
+            contextVersion: params.contextVersion,
+        })
+        return isString(result) ? result : JSON.stringify(result)
+    })
+}
+
+async function resolveSingleToken(params: ResolveSingleTokenParams): Promise<unknown> {
+    const { variableName, currentState } = params
+    const isConnection = variableName.startsWith(CONNECTIONS)
+    if (isConnection) {
+        return handleConnection(params)
+    }
+    return evalInScope(variableName, { ...currentState }, { flattenNestedKeys })
+}
+
+async function handleConnection(params: ResolveSingleTokenParams): Promise<unknown> {
+    const { variableName, engineToken, projectId, apiUrl, censoredInput } = params
+    const connectionName = parseConnectionNameOnly(variableName)
+    if (isNil(connectionName)) {
+        return ''
+    }
+    if (censoredInput) {
+        return '**REDACTED**'
+    }
+    const connection = await createConnectionService({ engineToken, projectId, apiUrl, contextVersion: params.contextVersion }).obtain(connectionName)
+    const pathAfterConnectionName = parsePathAfterConnectionName(variableName, connectionName)
+    if (isNil(pathAfterConnectionName) || pathAfterConnectionName.length === 0) {
+        return connection
+    }
+    return evalInScope(pathAfterConnectionName, { connection }, { flattenNestedKeys })
+}
+
+function parsePathAfterConnectionName(variableName: string, connectionName: string): string | null {
+    if (variableName.includes('[')) {
+        return variableName.substring(`connections.['${connectionName}']`.length)
+    }
+    const cp = variableName.substring(`connections.${connectionName}`.length)
+    if (cp.length === 0) {
+        return cp
+    }
+    return `connection${cp}`
+}
+
+function parseConnectionNameOnly(variableName: string): string | null {
+    const connectionWithNewFormatSquareBrackets = variableName.includes('[')
+    if (connectionWithNewFormatSquareBrackets) {
+        return parseSquareBracketConnectionPath(variableName)
+    }
+    // {{connections.connectionName.path}}
+    // This does not work If connectionName contains .
+    return variableName.split('.')?.[1]
+}
+
+function parseSquareBracketConnectionPath(variableName: string): string | null {
+    // Find the connection name inside {{connections['connectionName'].path}}
+    const matches = variableName.match(/\['([^']+)'\]/g)
+    if (matches && matches.length >= 1) {
+        // Remove the square brackets and quotes from the connection name
+
+        const secondPath = matches[0].replace(/\['|'\]/g, '')
+        return secondPath
+    }
+    return null
+}
+
+// eslint-disable-next-line @typescript-eslint/ban-types
+async function evalInScope(js: string, contextAsScope: Record<string, unknown>, functions: Record<string, Function>): Promise<unknown> {
+    const { data: result, error: resultError } = await utils.tryCatchAndThrowOnEngineError((async () => {
+        const codeSandbox = await initCodeSandbox()
+
+        const result = await codeSandbox.runScript({
+            script: js,
+            scriptContext: contextAsScope,
+            functions,
+        })
+        return result ?? ''
+    }))
+
+    if (resultError) {
+        console.warn('[evalInScope] Error evaluating variable', resultError)
+        return ''
+    }
+    return result ?? ''
+}
+
+function flattenNestedKeys(data: unknown, pathToMatch: string[]): unknown[] {
+    if (typeof data === 'object' && data !== null && !Array.isArray(data)) {
+        for (const [key, value] of Object.entries(data as Record<string, unknown>)) {
+            if (key === pathToMatch[0]) {
+                return flattenNestedKeys(value, pathToMatch.slice(1))
+            }
+        }
+    }
+    else if (Array.isArray(data)) {
+        return data.flatMap((d) => flattenNestedKeys(d, pathToMatch))
+    }
+    else if (pathToMatch.length === 0) {
+        return [data]
+    }
+    return []
+}
+
+type ResolveSingleTokenParams = {
+    variableName: string
+    currentState: Record<string, unknown>
+    engineToken: string
+    projectId: string
+    apiUrl: string
+    censoredInput: boolean
+    contextVersion: ContextVersion | undefined
+}
+
+type ResolveInputInternalParams = {
+    input: string
+    engineToken: string
+    projectId: string
+    apiUrl: string
+    censoredInput: boolean
+    currentState: Record<string, unknown>
+    contextVersion: ContextVersion | undefined
+}
+
+type ResolveInputParams = {
+    unresolvedInput: unknown
+    executionState: FlowExecutorContext
+}
+
+type ResolveResult<T = unknown> = {
+    resolvedInput: T
+    censoredInput: unknown
+}
+
+type PropsResolverParams = {
+    engineToken: string
+    projectId: string
+    apiUrl: string
+    contextVersion: ContextVersion | undefined
+}

activepieces-fork/packages/engine/src/lib/worker-socket.ts

@@ -0,0 +1,110 @@
+import { inspect } from 'util'
+import {
+    emitWithAck,
+    EngineGenericError,
+    EngineOperation,
+    EngineOperationType,
+    EngineResponse,
+    EngineResponseStatus,
+    EngineSocketEvent,
+    EngineStderr,
+    EngineStdout,
+    ERROR_MESSAGES_TO_REDACT,
+    isNil,
+} from '@activepieces/shared'
+import { io, type Socket } from 'socket.io-client'
+import { execute } from './operations'
+import { utils } from './utils'
+
+const WORKER_ID = process.env.WORKER_ID
+const WS_URL = 'ws://127.0.0.1:12345'
+
+let socket: Socket | undefined
+
+async function executeFromSocket(operation: EngineOperation, operationType: EngineOperationType): Promise<void> {
+    const result = await execute(operationType, operation)
+    const resultParsed = JSON.parse(JSON.stringify(result))
+    await workerSocket.sendToWorkerWithAck(EngineSocketEvent.ENGINE_RESPONSE, resultParsed)
+}
+
+export const workerSocket = {
+    init: (): void => {
+        if (isNil(WORKER_ID)) {
+            throw new EngineGenericError('WorkerIdNotSetError', 'WORKER_ID environment variable is not set')
+        }
+
+        socket = io(WS_URL, {
+            path: '/worker/ws',
+            auth: {
+                workerId: WORKER_ID,
+            },
+            autoConnect: true,
+            reconnection: true,
+        })
+
+        // Redirect console.log/error to socket
+        const originalLog = console.log
+        console.log = function (...args): void {
+            const engineStdout: EngineStdout = {
+                message: args.join(' ') + '\n',
+            }
+            socket?.emit(EngineSocketEvent.ENGINE_STDOUT, engineStdout)
+            originalLog.apply(console, args)
+        }
+
+        const originalError = console.error
+        console.error = function (...args): void {
+            let sanitizedArgs = [...args]
+            if (typeof args[0] === 'string' && ERROR_MESSAGES_TO_REDACT.some(errorMessage => args[0].includes(errorMessage))) {
+                sanitizedArgs = [sanitizedArgs[0], 'REDACTED']
+            }
+            const engineStderr: EngineStderr = {
+                message: sanitizedArgs.join(' ') + '\n',
+            }
+            socket?.emit(EngineSocketEvent.ENGINE_STDERR, engineStderr)
+
+            originalError.apply(console, sanitizedArgs)
+        }
+
+        socket.on(EngineSocketEvent.ENGINE_OPERATION, async (data: { operation: EngineOperation, operationType: EngineOperationType }) => {
+            const { error: resultError } = await utils.tryCatchAndThrowOnEngineError(() =>
+                executeFromSocket(data.operation, data.operationType),
+            )
+
+            if (resultError) {
+                const engineError: EngineResponse = {
+                    response: undefined,
+                    status: EngineResponseStatus.INTERNAL_ERROR,
+                    error: utils.formatExecutionError(resultError),
+                }
+                console.error(utils.formatExecutionError(resultError))
+                await workerSocket.sendToWorkerWithAck(EngineSocketEvent.ENGINE_RESPONSE, engineError)
+            }
+        })
+
+
+    },
+
+    sendToWorkerWithAck: async (
+        type: EngineSocketEvent,
+        data: unknown,
+    ): Promise<void> => {
+        await emitWithAck(socket, type, data, {
+            timeoutMs: 4000,
+            retries: 4,
+            retryDelayMs: 1000,
+        })
+    },
+
+    sendError: async (error: unknown): Promise<void> => {
+        const engineStderr: EngineStderr = {
+            message: inspect(error),
+        }
+        await emitWithAck(socket, EngineSocketEvent.ENGINE_STDERR, engineStderr, {
+            timeoutMs: 3000,
+            retries: 4,
+            retryDelayMs: 1000,
+        })
+    },
+}
+

activepieces-fork/packages/engine/src/main.ts

@@ -0,0 +1,21 @@
+import { isNil } from '@activepieces/shared'
+import { workerSocket } from './lib/worker-socket'
+
+const WORKER_ID = process.env.WORKER_ID
+process.title = `engine-${WORKER_ID}`
+
+if (!isNil(WORKER_ID)) {
+    workerSocket.init()
+}
+
+process.on('uncaughtException', (error) => {
+    void workerSocket.sendError(error).catch().finally(() => {
+        process.exit(3)
+    })
+})
+
+process.on('unhandledRejection', (reason) => {
+    void workerSocket.sendError(reason).catch().finally(() => {
+        process.exit(4)
+    })
+})