Add All Folders

Dagster/API_R&D/api/workflow.py

@@ -0,0 +1,96 @@
+from typing import List
+from api.task import Task, Status, TaskEvent
+from api.execution_context import ExecutionContext
+from api.io_port import InPort, CompatibilityException
+
+
+class Workflow(Task):
+    """
+    Composite node: contains children Tasks.
+    Executes tasks in dependency order (topological execution).
+    """
+
+    def __init__(self, name: str):
+        super().__init__(name)
+        self.children: List[Task] = []
+
+    def execute(self, context: ExecutionContext):
+        pass
+
+    def add_child(self, task: Task):
+        task.parent_task = self
+        self.children.append(task)
+
+    def remove_child(self, task: Task):
+        if task in self.children:
+            self.children.remove(task)
+            task.parent_task = None
+
+    def get_children(self):
+        return list(self.children)
+
+    def get_all_tasks(self):
+        tasks = []
+        for child in self.children:
+            tasks.append(child)
+            if isinstance(child, Workflow):
+                tasks.extend(child.get_all_tasks())
+        return tasks
+
+    def validate(self):
+        super().validate()
+        for child in self.children:
+            child.validate()
+
+    def _get_dependencies(self, task: Task) -> List[Task]:
+        deps = []
+        for in_port in task.in_ports:
+            if isinstance(in_port, InPort) and in_port.connected_output:
+                upstream_task = in_port.connected_output.parent_task
+                if upstream_task and upstream_task != task:
+                    deps.append(upstream_task)
+        return deps
+
+    def run(self, context: ExecutionContext):
+        self._set_status(Status.RUNNING)
+        self._emit(TaskEvent.ON_START, {"workflow": self.name})
+
+        remaining = set(self.children)
+        completed = set()
+
+        while remaining:
+            progress_made = False
+
+            for task in list(remaining):
+                deps = self._get_dependencies(task)
+
+                if all(dep in completed for dep in deps):
+                    if task.check_inputs_ready(context):
+                        try:
+                            task.run(context)
+                            completed.add(task)
+                            remaining.remove(task)
+                            progress_made = True
+
+                        except CompatibilityException as e:
+                            task._set_status(Status.FAILED)
+                            self._emit(TaskEvent.ON_ERROR, {"error": e.reason, "task": task.name})
+                            self._set_status(Status.FAILED)
+                            return
+
+                        except Exception as e:
+                            task._set_status(Status.FAILED)
+                            self._emit(TaskEvent.ON_ERROR, {"error": str(e), "task": task.name})
+                            self._set_status(Status.FAILED)
+                            return
+
+            if not progress_made:
+                self._set_status(Status.FAILED)
+                self._emit(TaskEvent.ON_ERROR, {"error": "Dependency deadlock detected"})
+                return
+
+        if all(t.status == Status.COMPLETED for t in self.children):
+            self._set_status(Status.COMPLETED)
+            self._emit(TaskEvent.ON_FINISH, {"workflow": self.name})
+        else:
+            self._set_status(Status.FAILED)