Add All Folders
This commit is contained in:
soorena62
@@ -0,0 +1,8 @@
|
||||
Description:
|
||||
# This project was created to test API layer codes in depth, focusing on all the capabilities, features, and methods implemented in the codes, such as dynamic type checking and type detection at the moment of data entry, connecting output ports to input, etc. in a real environment. This project has a GUI and allows you to create a workflow, select the required nodes, run the workflow, connect nodes, and manually.
|
||||
|
||||
Dependencies:
|
||||
# pip install dagster pysera pandas pyside6 numpy
|
||||
|
||||
Running:
|
||||
# python -m gui.app
|
||||
@@ -0,0 +1,45 @@
|
||||
from typing import Any, Dict, Optional
|
||||
from copy import deepcopy
|
||||
from api.io_port import DType
|
||||
|
||||
|
||||
class Asset:
|
||||
"""
|
||||
Represents the output of a Task.
|
||||
|
||||
Stores:
|
||||
- data: actual payload
|
||||
- dtype: semantic type (DType instance)
|
||||
- metadata: optional dictionary
|
||||
"""
|
||||
|
||||
def __init__(self, data: Any, dtype: Optional[DType] = None, metadata: Optional[Dict[str, Any]] = None):
|
||||
self.data = data
|
||||
self.dtype = dtype # DO NOT deepcopy dtype
|
||||
self.metadata = metadata or {}
|
||||
|
||||
# Whether this asset should be preserved after workflow execution
|
||||
self.preserved = True
|
||||
|
||||
# Lifecycle
|
||||
def dismiss(self):
|
||||
"""Mark asset as disposable."""
|
||||
self.preserved = False
|
||||
|
||||
# Utility
|
||||
def clone(self) -> "Asset":
|
||||
"""
|
||||
Create a deep copy of the asset.
|
||||
dtype is NOT deepcopied because DType objects are not deepcopy-safe.
|
||||
"""
|
||||
return Asset(
|
||||
data=deepcopy(self.data),
|
||||
dtype=self.dtype, # keep reference
|
||||
metadata=deepcopy(self.metadata)
|
||||
)
|
||||
|
||||
# Representation
|
||||
def __repr__(self):
|
||||
# Avoid printing full dtype object (may contain nested structures)
|
||||
dtype_name = self.dtype.__class__.__name__ if self.dtype else "None"
|
||||
return f"<Asset dtype={dtype_name} preserved={self.preserved}>"
|
||||
@@ -0,0 +1,82 @@
|
||||
from typing import Dict, Any
|
||||
from api.assets import Asset
|
||||
from api.io_port import InPort, OutPort
|
||||
|
||||
|
||||
class ExecutionContext:
|
||||
"""
|
||||
Runtime data container for a workflow execution.
|
||||
Stores produced assets for OutPorts and allows InPorts to retrieve them.
|
||||
"""
|
||||
|
||||
def __init__(self, execution_id: str | None = None):
|
||||
self.execution_id = execution_id
|
||||
|
||||
# Key: OutPort, Value: Asset
|
||||
self._data: Dict[OutPort, Asset] = {}
|
||||
|
||||
# Arbitrary metadata (optional)
|
||||
self._metadata: Dict[str, Any] = {}
|
||||
|
||||
# Store produced data
|
||||
def put(self, output_port: OutPort, asset: Asset):
|
||||
"""
|
||||
Register an asset produced by an OutPort.
|
||||
Also updates output_port.produced_asset.
|
||||
"""
|
||||
self._data[output_port] = asset
|
||||
output_port.produced_asset = asset
|
||||
|
||||
# Retrieve data
|
||||
def get(self, port):
|
||||
"""
|
||||
Retrieve asset for:
|
||||
- OutPort → return its asset
|
||||
- InPort → return asset from its connected OutPort
|
||||
"""
|
||||
if isinstance(port, OutPort):
|
||||
return self._data.get(port)
|
||||
|
||||
if isinstance(port, InPort):
|
||||
if port.connected_output is None:
|
||||
return None
|
||||
return self._data.get(port.connected_output)
|
||||
|
||||
raise TypeError("ExecutionContext.get() expects InPort or OutPort")
|
||||
|
||||
# Check if data exists
|
||||
def has_data(self, port) -> bool:
|
||||
if isinstance(port, OutPort):
|
||||
return port in self._data
|
||||
|
||||
if isinstance(port, InPort):
|
||||
if port.connected_output is None:
|
||||
return False
|
||||
return port.connected_output in self._data
|
||||
|
||||
return False
|
||||
|
||||
# Metadata management
|
||||
def set_meta(self, key: str, value: Any):
|
||||
self._metadata[key] = value
|
||||
|
||||
def get_meta(self, key: str, default=None):
|
||||
return self._metadata.get(key, default)
|
||||
|
||||
# Debug snapshot
|
||||
def status_view(self):
|
||||
"""
|
||||
Lightweight snapshot of current execution state.
|
||||
Safe for GUI display.
|
||||
"""
|
||||
return {
|
||||
"execution_id": self.execution_id,
|
||||
"assets": {
|
||||
f"{port.parent_task.name}.{port.name}": {
|
||||
"dtype": port.dtype.__class__.__name__,
|
||||
"preserved": asset.preserved,
|
||||
}
|
||||
for port, asset in self._data.items()
|
||||
},
|
||||
"metadata": self._metadata.copy(),
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
|
||||
# Exceptions
|
||||
class CompatibilityException(Exception):
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
# DType (Semantic Contract)
|
||||
class DType(ABC):
|
||||
"""
|
||||
Pure semantic type. Does NOT contain names.
|
||||
Names belong to Ports or CompositePart.
|
||||
"""
|
||||
def __init__(self, metadata: Optional[Dict[str, Any]] = None):
|
||||
self.metadata = metadata or {}
|
||||
self.conditions: Optional[Dict[str, Any]] = None
|
||||
|
||||
@abstractmethod
|
||||
def can_connect_to(self, other: "DType", conditions: Optional[Dict[str, Any]] = None):
|
||||
pass
|
||||
|
||||
def set_conditions(self, conditions: Dict[str, Any]):
|
||||
self.conditions = conditions
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.__class__.__name__}(metadata={self.metadata})"
|
||||
|
||||
|
||||
# Composite Types
|
||||
class CompositePart:
|
||||
"""
|
||||
Represents one named component inside a CompositeType.
|
||||
Example:
|
||||
CompositePart("image", ImageType("nifti"))
|
||||
"""
|
||||
def __init__(self, name: str, dtype: DType):
|
||||
self.name = name
|
||||
self.dtype = dtype
|
||||
|
||||
def __repr__(self):
|
||||
return f"CompositePart(name={self.name}, dtype={self.dtype})"
|
||||
|
||||
|
||||
class CompositeType(DType):
|
||||
"""
|
||||
CompositeType is a LIST of CompositePart.
|
||||
Order matters. Names are preserved inside CompositePart.
|
||||
"""
|
||||
def __init__(self, parts: List[CompositePart], metadata=None):
|
||||
super().__init__(metadata)
|
||||
self.parts = parts
|
||||
|
||||
def can_connect_to(self, other: "DType", conditions=None):
|
||||
if not isinstance(other, CompositeType):
|
||||
raise CompatibilityException("Expected CompositeType")
|
||||
|
||||
if len(self.parts) != len(other.parts):
|
||||
raise CompatibilityException("CompositeType length mismatch")
|
||||
|
||||
for i in range(len(self.parts)):
|
||||
p1 = self.parts[i]
|
||||
p2 = other.parts[i]
|
||||
|
||||
if p1.name != p2.name:
|
||||
raise CompatibilityException(
|
||||
f"CompositeType part name mismatch: {p1.name} vs {p2.name}"
|
||||
)
|
||||
|
||||
p1.dtype.can_connect_to(p2.dtype, conditions)
|
||||
|
||||
def __repr__(self):
|
||||
return f"CompositeType(parts={self.parts})"
|
||||
|
||||
|
||||
# MaskType:
|
||||
class MaskType(DType):
|
||||
def __init__(self, metadata=None):
|
||||
super().__init__(metadata)
|
||||
|
||||
def can_connect_to(self, other: DType, conditions=None):
|
||||
if not isinstance(other, MaskType):
|
||||
raise CompatibilityException("Target is not a MaskType")
|
||||
|
||||
|
||||
# Image Types:
|
||||
class ImageType(DType):
|
||||
def __init__(self, modality: Optional[str] = None, metadata=None):
|
||||
super().__init__(metadata)
|
||||
self.modality = modality
|
||||
|
||||
def can_connect_to(self, other: DType, conditions=None):
|
||||
if not isinstance(other, ImageType):
|
||||
raise CompatibilityException("Target is not an ImageType")
|
||||
|
||||
if self.modality and other.modality and self.modality != other.modality:
|
||||
raise CompatibilityException(
|
||||
f"Image modality mismatch: {self.modality} vs {other.modality}"
|
||||
)
|
||||
|
||||
|
||||
class NIFTIImageType(ImageType):
|
||||
def __init__(self, metadata=None):
|
||||
super().__init__(modality="nifti", metadata=metadata)
|
||||
|
||||
|
||||
class DICOMImageType(ImageType):
|
||||
def __init__(self, metadata=None):
|
||||
super().__init__(modality="dicom", metadata=metadata)
|
||||
|
||||
|
||||
# Table Types:
|
||||
class TableType(DType):
|
||||
def __init__(self, columns: List[str], metadata=None):
|
||||
super().__init__(metadata)
|
||||
self.columns = columns
|
||||
|
||||
def can_connect_to(self, other: DType, conditions=None):
|
||||
if not isinstance(other, TableType):
|
||||
raise CompatibilityException("Target is not a TableType")
|
||||
|
||||
if self.columns != other.columns:
|
||||
raise CompatibilityException(
|
||||
f"Table schema mismatch: {self.columns} vs {other.columns}"
|
||||
)
|
||||
|
||||
|
||||
class CSVTableType(TableType):
|
||||
def __init__(self, columns: List[str], delimiter: str = ",", encoding: str = "utf-8", metadata=None):
|
||||
super().__init__(columns, metadata)
|
||||
self.delimiter = delimiter
|
||||
self.encoding = encoding
|
||||
|
||||
def can_connect_to(self, other: DType, conditions=None):
|
||||
if not isinstance(other, CSVTableType):
|
||||
raise CompatibilityException("CSV tables can only connect to CSV tables")
|
||||
|
||||
super().can_connect_to(other, conditions)
|
||||
|
||||
if self.delimiter != other.delimiter:
|
||||
raise CompatibilityException(
|
||||
f"CSV delimiter mismatch: {self.delimiter} vs {other.delimiter}"
|
||||
)
|
||||
|
||||
|
||||
# Text Types:
|
||||
class TextType(DType):
|
||||
def __init__(self, language: Optional[str] = None, encoding: Optional[str] = None, metadata=None):
|
||||
super().__init__(metadata)
|
||||
self.language = language
|
||||
self.encoding = encoding
|
||||
|
||||
def can_connect_to(self, other: DType, conditions=None):
|
||||
if not isinstance(other, TextType):
|
||||
raise CompatibilityException("Target is not TextType")
|
||||
|
||||
if self.language and other.language and self.language != other.language:
|
||||
raise CompatibilityException(
|
||||
f"Language mismatch: {self.language} vs {other.language}"
|
||||
)
|
||||
|
||||
if self.encoding and other.encoding and self.encoding != other.encoding:
|
||||
raise CompatibilityException(
|
||||
f"Encoding mismatch: {self.encoding} vs {other.encoding}"
|
||||
)
|
||||
|
||||
|
||||
# Ports:
|
||||
class Port(ABC):
|
||||
def __init__(self, name: str, dtype: DType):
|
||||
self.name = name
|
||||
self.dtype = dtype
|
||||
self.parent_task = None # Set by Workflow builder
|
||||
|
||||
def full_name(self):
|
||||
if self.parent_task:
|
||||
return f"{self.parent_task.name}.{self.name}"
|
||||
return self.name
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.__class__.__name__}({self.full_name()}, {self.dtype})"
|
||||
|
||||
|
||||
# OutPort (fan-out supported):
|
||||
class OutPort(Port):
|
||||
def __init__(self, name: str, dtype: DType):
|
||||
super().__init__(name, dtype)
|
||||
self._connections: List["InPort"] = []
|
||||
self.produced_asset = None # Set by ExecutionContext
|
||||
|
||||
@property
|
||||
def connections(self):
|
||||
return list(self._connections)
|
||||
|
||||
def connect(self, in_port: "InPort", conditions: Optional[Dict[str, Any]] = None):
|
||||
# Fan-out allowed → no limit on connections
|
||||
in_port.can_connect_to(self, conditions)
|
||||
|
||||
self._connections.append(in_port)
|
||||
in_port._connected_output = self
|
||||
|
||||
def disconnect(self, in_port: "InPort"):
|
||||
if in_port in self._connections:
|
||||
self._connections.remove(in_port)
|
||||
in_port._connected_output = None
|
||||
|
||||
def get_downstream_ports(self):
|
||||
return list(self._connections)
|
||||
|
||||
|
||||
# InPort:
|
||||
class InPort(Port):
|
||||
def __init__(self, name: str, dtype: DType, required: bool = True):
|
||||
super().__init__(name, dtype)
|
||||
self.required = required
|
||||
self._connected_output: Optional[OutPort] = None
|
||||
|
||||
@property
|
||||
def connected_output(self):
|
||||
return self._connected_output
|
||||
|
||||
def is_connected(self):
|
||||
return self._connected_output is not None
|
||||
|
||||
def is_ready(self, context):
|
||||
"""
|
||||
True only if data exists in ExecutionContext.
|
||||
"""
|
||||
if not self._connected_output:
|
||||
return False
|
||||
return context.has_data(self)
|
||||
|
||||
def can_connect_to(self, out_port: OutPort, conditions: Optional[Dict[str, Any]] = None):
|
||||
self.dtype.can_connect_to(out_port.dtype, conditions)
|
||||
|
||||
def disconnect(self):
|
||||
if self._connected_output:
|
||||
self._connected_output._connections.remove(self)
|
||||
self._connected_output = None
|
||||
|
||||
def get_upstream_port(self):
|
||||
return self._connected_output
|
||||
|
||||
def validate(self):
|
||||
if self.required and not self.is_connected():
|
||||
raise CompatibilityException(f"InPort {self.full_name()} is required but not connected")
|
||||
@@ -0,0 +1,89 @@
|
||||
from api.task import Task, Status, TaskEvent
|
||||
from api.assets import Asset
|
||||
from api.execution_context import ExecutionContext
|
||||
from api.io_port import CompatibilityException, InPort, OutPort
|
||||
|
||||
|
||||
class Module(Task):
|
||||
"""
|
||||
Leaf node: atomic executable unit.
|
||||
Executes using its input ports and produces assets on output ports.
|
||||
"""
|
||||
|
||||
def add_in_port(self, port: InPort):
|
||||
port.parent_task = self
|
||||
self.in_ports.append(port)
|
||||
|
||||
def add_out_port(self, port: OutPort):
|
||||
port.parent_task = self
|
||||
self.out_ports.append(port)
|
||||
|
||||
# Hooks
|
||||
def before_run(self, context: ExecutionContext):
|
||||
pass
|
||||
|
||||
def after_run(self, context: ExecutionContext):
|
||||
pass
|
||||
|
||||
def on_error(self, error: Exception):
|
||||
pass
|
||||
|
||||
# Validation
|
||||
def validate(self):
|
||||
super().validate()
|
||||
|
||||
# Main execution entry point
|
||||
def run(self, context: ExecutionContext):
|
||||
self.before_run(context)
|
||||
|
||||
if not self.check_inputs_ready(context):
|
||||
self._set_status(Status.PENDING)
|
||||
return
|
||||
|
||||
self._set_status(Status.RUNNING)
|
||||
self._emit(TaskEvent.ON_START, {"module": self.name})
|
||||
|
||||
try:
|
||||
# Gather input data
|
||||
consumed_data = {}
|
||||
for port in self.in_ports:
|
||||
if not port.is_ready(context):
|
||||
raise CompatibilityException(
|
||||
f"Input {port.full_name()} is not ready"
|
||||
)
|
||||
asset = context.get(port)
|
||||
consumed_data[port.name] = asset.data
|
||||
|
||||
# Execute module logic
|
||||
result = self.execute(consumed_data)
|
||||
|
||||
if not isinstance(result, dict):
|
||||
raise ValueError(f"Module '{self.name}' must return dict")
|
||||
|
||||
# Store outputs
|
||||
for out_port in self.out_ports:
|
||||
if out_port.name not in result:
|
||||
raise ValueError(
|
||||
f"Output '{out_port.name}' missing in module '{self.name}' result"
|
||||
)
|
||||
|
||||
asset = Asset(
|
||||
data=result[out_port.name],
|
||||
dtype=out_port.dtype
|
||||
)
|
||||
|
||||
context.put(out_port, asset)
|
||||
|
||||
self._set_status(Status.COMPLETED)
|
||||
self._emit(TaskEvent.ON_FINISH, {"module": self.name})
|
||||
self.after_run(context)
|
||||
|
||||
except Exception as e:
|
||||
self._set_status(Status.FAILED)
|
||||
self.on_error(e)
|
||||
self._emit(TaskEvent.ON_ERROR, {"error": str(e)})
|
||||
raise e
|
||||
|
||||
# Default execute
|
||||
def execute(self, inputs):
|
||||
return inputs
|
||||
@@ -0,0 +1,64 @@
|
||||
from typing import Set
|
||||
from api.task import Task, Status
|
||||
from api.execution_context import ExecutionContext
|
||||
|
||||
|
||||
class Scheduler:
|
||||
"""
|
||||
DAG-based execution scheduler.
|
||||
Decides WHEN a Task can run based on port readiness and orchestrates execution.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._completed: Set[Task] = set()
|
||||
self._failed: Set[Task] = set()
|
||||
|
||||
def run(self, root: Task, context: ExecutionContext):
|
||||
self._execute_task(root, context)
|
||||
|
||||
def _execute_task(self, task: Task, context: ExecutionContext):
|
||||
if task.status in (Status.COMPLETED, Status.FAILED, Status.STOPPED):
|
||||
return
|
||||
|
||||
if hasattr(task, "children"):
|
||||
self._run_workflow(task, context)
|
||||
return
|
||||
|
||||
if not task.check_inputs_ready(context):
|
||||
print(f"[Scheduler] Task '{task.name}' not ready, skipping for now.")
|
||||
return
|
||||
|
||||
try:
|
||||
task._set_status(Status.RUNNING)
|
||||
task.run(context)
|
||||
task._set_status(Status.COMPLETED)
|
||||
self._completed.add(task)
|
||||
except Exception as e:
|
||||
task._set_status(Status.FAILED)
|
||||
self._failed.add(task)
|
||||
print(f"[Scheduler] Task '{task.name}' failed: {e}")
|
||||
raise
|
||||
|
||||
def _run_workflow(self, workflow: Task, context: ExecutionContext):
|
||||
workflow._set_status(Status.RUNNING)
|
||||
|
||||
remaining = set(workflow.children)
|
||||
|
||||
while remaining:
|
||||
progress_made = False
|
||||
|
||||
for child in list(remaining):
|
||||
if child.check_inputs_ready(context):
|
||||
self._execute_task(child, context)
|
||||
remaining.remove(child)
|
||||
progress_made = True
|
||||
|
||||
if not progress_made:
|
||||
raise RuntimeError(
|
||||
f"Workflow '{workflow.name}' deadlock: unresolved dependencies"
|
||||
)
|
||||
|
||||
if any(c.status == Status.FAILED for c in workflow.children):
|
||||
workflow._set_status(Status.FAILED)
|
||||
else:
|
||||
workflow._set_status(Status.COMPLETED)
|
||||
@@ -0,0 +1,78 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from enum import Enum
|
||||
from typing import List, Any, Optional
|
||||
|
||||
from api.io_port import InPort, OutPort
|
||||
from api.execution_context import ExecutionContext
|
||||
|
||||
|
||||
class Status(Enum):
|
||||
PENDING = "pending"
|
||||
READY = "ready"
|
||||
RUNNING = "running"
|
||||
PAUSED = "paused"
|
||||
STOPPED = "stopped"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class TaskEvent(Enum):
|
||||
BEFORE_RUN = "before_run"
|
||||
AFTER_RUN = "after_run"
|
||||
ON_ERROR = "on_error"
|
||||
STATUS_CHANGED = "status_changed"
|
||||
ON_START = "on_start"
|
||||
ON_FINISH = "on_finish"
|
||||
|
||||
|
||||
class TaskEventListener:
|
||||
def handle(self, event: TaskEvent, task: "Task", payload: Optional[Any] = None):
|
||||
pass
|
||||
|
||||
|
||||
class Task(ABC):
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.status = Status.PENDING
|
||||
|
||||
self.in_ports: List[InPort] = []
|
||||
self.out_ports: List[OutPort] = []
|
||||
|
||||
self._listeners: List[TaskEventListener] = []
|
||||
self.parent_task: Optional["Task"] = None
|
||||
|
||||
# Event System
|
||||
def add_listener(self, listener: TaskEventListener):
|
||||
self._listeners.append(listener)
|
||||
|
||||
def _emit(self, event: TaskEvent, payload: Optional[Any] = None):
|
||||
for listener in self._listeners:
|
||||
listener.handle(event, self, payload)
|
||||
|
||||
def _set_status(self, new_status: Status):
|
||||
old_status = self.status
|
||||
self.status = new_status
|
||||
|
||||
if old_status != new_status:
|
||||
self._emit(TaskEvent.STATUS_CHANGED, {"from": old_status, "to": new_status})
|
||||
|
||||
# Input Readiness Check
|
||||
def check_inputs_ready(self, context: ExecutionContext) -> bool:
|
||||
for port in self.in_ports:
|
||||
if not port.is_ready(context):
|
||||
return False
|
||||
return True
|
||||
|
||||
# Execution Wrapper
|
||||
def run(self, context: ExecutionContext):
|
||||
"""
|
||||
Base Task.run() SHOULD NOT be used for Module.
|
||||
Module overrides run() completely.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Task.run() should not be called for Module. Use Module.run() instead."
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def execute(self, context: ExecutionContext):
|
||||
pass
|
||||
@@ -0,0 +1,96 @@
|
||||
from typing import List
|
||||
from api.task import Task, Status, TaskEvent
|
||||
from api.execution_context import ExecutionContext
|
||||
from api.io_port import InPort, CompatibilityException
|
||||
|
||||
|
||||
class Workflow(Task):
|
||||
"""
|
||||
Composite node: contains children Tasks.
|
||||
Executes tasks in dependency order (topological execution).
|
||||
"""
|
||||
|
||||
def __init__(self, name: str):
|
||||
super().__init__(name)
|
||||
self.children: List[Task] = []
|
||||
|
||||
def execute(self, context: ExecutionContext):
|
||||
pass
|
||||
|
||||
def add_child(self, task: Task):
|
||||
task.parent_task = self
|
||||
self.children.append(task)
|
||||
|
||||
def remove_child(self, task: Task):
|
||||
if task in self.children:
|
||||
self.children.remove(task)
|
||||
task.parent_task = None
|
||||
|
||||
def get_children(self):
|
||||
return list(self.children)
|
||||
|
||||
def get_all_tasks(self):
|
||||
tasks = []
|
||||
for child in self.children:
|
||||
tasks.append(child)
|
||||
if isinstance(child, Workflow):
|
||||
tasks.extend(child.get_all_tasks())
|
||||
return tasks
|
||||
|
||||
def validate(self):
|
||||
super().validate()
|
||||
for child in self.children:
|
||||
child.validate()
|
||||
|
||||
def _get_dependencies(self, task: Task) -> List[Task]:
|
||||
deps = []
|
||||
for in_port in task.in_ports:
|
||||
if isinstance(in_port, InPort) and in_port.connected_output:
|
||||
upstream_task = in_port.connected_output.parent_task
|
||||
if upstream_task and upstream_task != task:
|
||||
deps.append(upstream_task)
|
||||
return deps
|
||||
|
||||
def run(self, context: ExecutionContext):
|
||||
self._set_status(Status.RUNNING)
|
||||
self._emit(TaskEvent.ON_START, {"workflow": self.name})
|
||||
|
||||
remaining = set(self.children)
|
||||
completed = set()
|
||||
|
||||
while remaining:
|
||||
progress_made = False
|
||||
|
||||
for task in list(remaining):
|
||||
deps = self._get_dependencies(task)
|
||||
|
||||
if all(dep in completed for dep in deps):
|
||||
if task.check_inputs_ready(context):
|
||||
try:
|
||||
task.run(context)
|
||||
completed.add(task)
|
||||
remaining.remove(task)
|
||||
progress_made = True
|
||||
|
||||
except CompatibilityException as e:
|
||||
task._set_status(Status.FAILED)
|
||||
self._emit(TaskEvent.ON_ERROR, {"error": e.reason, "task": task.name})
|
||||
self._set_status(Status.FAILED)
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
task._set_status(Status.FAILED)
|
||||
self._emit(TaskEvent.ON_ERROR, {"error": str(e), "task": task.name})
|
||||
self._set_status(Status.FAILED)
|
||||
return
|
||||
|
||||
if not progress_made:
|
||||
self._set_status(Status.FAILED)
|
||||
self._emit(TaskEvent.ON_ERROR, {"error": "Dependency deadlock detected"})
|
||||
return
|
||||
|
||||
if all(t.status == Status.COMPLETED for t in self.children):
|
||||
self._set_status(Status.COMPLETED)
|
||||
self._emit(TaskEvent.ON_FINISH, {"workflow": self.name})
|
||||
else:
|
||||
self._set_status(Status.FAILED)
|
||||
@@ -0,0 +1,66 @@
|
||||
from dagster import op, job, In, Out
|
||||
import pysera
|
||||
from api.execution_context import ExecutionContext
|
||||
from api.assets import Asset
|
||||
|
||||
|
||||
@op(out={"image_input": Out(str), "mask_input": Out(str)})
|
||||
def image_reader_op():
|
||||
image_input = "data/sample/images"
|
||||
mask_input = "data/sample/masks"
|
||||
return {"image_input": image_input, "mask_input": mask_input}
|
||||
|
||||
|
||||
@op(ins={"image_input": In(str), "mask_input": In(str)}, out={"features": Out(Asset)})
|
||||
def pysera_extractor_op(image_input, mask_input):
|
||||
output_dir = "results"
|
||||
|
||||
result = pysera.process_batch(
|
||||
image_input=image_input,
|
||||
mask_input=mask_input,
|
||||
output_path=output_dir,
|
||||
num_workers="auto",
|
||||
enable_parallelism=True,
|
||||
apply_preprocessing=True,
|
||||
categories="all",
|
||||
dimensions="1st,2_5d,3d",
|
||||
feature_value_mode="REAL_VALUE",
|
||||
extraction_mode="handcrafted_feature",
|
||||
report="info",
|
||||
)
|
||||
|
||||
df = result.get("features_extracted")
|
||||
return {"features": Asset(df, "pysera_features_df")}
|
||||
|
||||
@op(ins={"features": In(Asset)}, out={"csv_path": Out(Asset)})
|
||||
def csv_writer_op(features):
|
||||
import os
|
||||
import csv
|
||||
from datetime import datetime
|
||||
|
||||
df = features.data
|
||||
rows = []
|
||||
if hasattr(df, "iterrows"):
|
||||
for _, row in df.iterrows():
|
||||
rows.append({"name": row[0], "value": row[1]})
|
||||
|
||||
os.makedirs("results", exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
path = os.path.join("results", f"features_dagster_{ts}.csv")
|
||||
|
||||
with open(path, "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["name", "value"])
|
||||
for r in rows:
|
||||
w.writerow([r["name"], r["value"]])
|
||||
|
||||
return {"csv_path": Asset(path, "csv_path")}
|
||||
|
||||
@job
|
||||
def radiuma_pysera_job():
|
||||
image_mask = image_reader_op()
|
||||
feats = pysera_extractor_op(
|
||||
image_input=image_mask["image_input"],
|
||||
mask_input=image_mask["mask_input"],
|
||||
)
|
||||
csv_writer_op(features=feats["features"])
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,21 @@
|
||||
import sys
|
||||
from PySide6.QtWidgets import QApplication
|
||||
from gui.node_editor import NodeEditor
|
||||
from modules.image_reader import factory as image_reader_factory
|
||||
from modules.extractor import factory as pysera_factory
|
||||
from modules.writer import factory as csv_factory
|
||||
|
||||
def main():
|
||||
app = QApplication(sys.argv)
|
||||
catalog = {
|
||||
"ImageReader": image_reader_factory,
|
||||
"PySERAExtractor": pysera_factory,
|
||||
"CSVWriter": csv_factory
|
||||
}
|
||||
editor = NodeEditor(catalog)
|
||||
editor.resize(1100, 720)
|
||||
editor.show()
|
||||
sys.exit(app.exec())
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,460 @@
|
||||
import time
|
||||
from PySide6.QtWidgets import (
|
||||
QWidget, QGraphicsView, QGraphicsScene, QListWidget, QListWidgetItem,
|
||||
QPushButton, QHBoxLayout, QVBoxLayout, QDialog, QTextEdit
|
||||
)
|
||||
from PySide6.QtGui import (
|
||||
QPainter, QPen, QColor, QPainterPath, QTransform, QMouseEvent
|
||||
)
|
||||
from PySide6.QtCore import Qt, QRectF
|
||||
from PySide6.QtWidgets import QGraphicsItem
|
||||
|
||||
from api.execution_context import ExecutionContext
|
||||
from api.workflow import Workflow
|
||||
from api.scheduler import Scheduler
|
||||
|
||||
|
||||
# PORT ITEM
|
||||
class PortItem(QGraphicsItem):
|
||||
R = 6
|
||||
|
||||
def __init__(self, parent_node, name, is_output, index):
|
||||
super().__init__(parent_node)
|
||||
self.parent_node = parent_node
|
||||
self.name = name
|
||||
self.is_output = is_output
|
||||
self.index = index
|
||||
self.setFlag(QGraphicsItem.ItemIsSelectable)
|
||||
|
||||
def boundingRect(self):
|
||||
return QRectF(-self.R, -self.R, 2*self.R, 2*self.R)
|
||||
|
||||
def paint(self, painter, option, widget=None):
|
||||
painter.setRenderHint(QPainter.Antialiasing)
|
||||
color = QColor(34,139,34) if self.is_output else QColor(70,130,180)
|
||||
painter.setPen(QPen(Qt.black, 1))
|
||||
painter.setBrush(color)
|
||||
painter.drawEllipse(self.boundingRect())
|
||||
|
||||
|
||||
# CONNECTION ITEM
|
||||
class ConnectionItem(QGraphicsItem):
|
||||
def __init__(self, out_port, in_port, editor):
|
||||
super().__init__()
|
||||
self.out_port = out_port
|
||||
self.in_port = in_port
|
||||
self.editor = editor
|
||||
self.setZValue(-1)
|
||||
self._rect = QRectF()
|
||||
self._path = QPainterPath()
|
||||
self.setFlag(QGraphicsItem.ItemIsSelectable)
|
||||
|
||||
def update_path(self):
|
||||
p1 = self.out_port.scenePos()
|
||||
p2 = self.in_port.scenePos()
|
||||
|
||||
new_rect = QRectF(p1, p2).normalized().adjusted(-20, -20, 20, 20)
|
||||
if new_rect != self._rect:
|
||||
self.prepareGeometryChange()
|
||||
self._rect = new_rect
|
||||
|
||||
path = QPainterPath(p1)
|
||||
dx = (p2.x() - p1.x()) * 0.5
|
||||
path.cubicTo(p1.x() + dx, p1.y(), p2.x() - dx, p2.y(), p2.x(), p2.y())
|
||||
self._path = path
|
||||
|
||||
def boundingRect(self):
|
||||
return self._rect
|
||||
|
||||
def paint(self, painter, option, widget=None):
|
||||
self.update_path()
|
||||
painter.setRenderHint(QPainter.Antialiasing)
|
||||
|
||||
if self.isSelected():
|
||||
painter.setPen(QPen(QColor(255, 0, 0), 3))
|
||||
else:
|
||||
painter.setPen(QPen(QColor(50, 50, 50), 2))
|
||||
|
||||
painter.drawPath(self._path)
|
||||
|
||||
def mousePressEvent(self, event):
|
||||
if event.button() == Qt.LeftButton:
|
||||
self.editor._remove_connection(self)
|
||||
super().mousePressEvent(event)
|
||||
|
||||
|
||||
# NODE ITEM
|
||||
class NodeItem(QGraphicsItem):
|
||||
W = 200
|
||||
H = 80
|
||||
|
||||
def __init__(self, module, title, editor):
|
||||
super().__init__()
|
||||
self.module = module
|
||||
self.title = title
|
||||
self.editor = editor
|
||||
self.running = False
|
||||
|
||||
self.in_ports = []
|
||||
self.out_ports = []
|
||||
|
||||
self.setFlags(QGraphicsItem.ItemIsMovable | QGraphicsItem.ItemIsSelectable)
|
||||
self._build_ports()
|
||||
|
||||
def highlight(self, active):
|
||||
self.running = active
|
||||
self.update()
|
||||
|
||||
def _build_ports(self):
|
||||
y = 30
|
||||
for i, p in enumerate(self.module.in_ports):
|
||||
port = PortItem(self, p.name, False, i)
|
||||
port.setPos(0, y)
|
||||
self.in_ports.append(port)
|
||||
y += 20
|
||||
|
||||
y = 30
|
||||
for i, p in enumerate(self.module.out_ports):
|
||||
port = PortItem(self, p.name, True, i)
|
||||
port.setPos(self.W, y)
|
||||
self.out_ports.append(port)
|
||||
y += 20
|
||||
|
||||
def boundingRect(self):
|
||||
return QRectF(0, 0, self.W, self.H)
|
||||
|
||||
def paint(self, painter, option, widget=None):
|
||||
painter.setRenderHint(QPainter.Antialiasing)
|
||||
|
||||
if self.running:
|
||||
painter.setBrush(QColor("#c8f7c5"))
|
||||
else:
|
||||
painter.setBrush(QColor("#f0f0f0"))
|
||||
|
||||
painter.setPen(QPen(Qt.black, 1))
|
||||
painter.drawRect(self.boundingRect())
|
||||
painter.drawText(10, 20, self.title)
|
||||
|
||||
def itemChange(self, change, value):
|
||||
if change == QGraphicsItem.ItemPositionHasChanged:
|
||||
self.editor._update_connections_for_node(self)
|
||||
return super().itemChange(change, value)
|
||||
|
||||
|
||||
# CUSTOM VIEW
|
||||
class NodeGraphicsView(QGraphicsView):
|
||||
def __init__(self, scene, editor):
|
||||
super().__init__(scene)
|
||||
self.editor = editor
|
||||
self.setRenderHint(QPainter.Antialiasing)
|
||||
|
||||
def mousePressEvent(self, event: QMouseEvent):
|
||||
pos = self.mapToScene(event.pos())
|
||||
item = self.scene().itemAt(pos, QTransform())
|
||||
|
||||
port = self.editor._find_port(item)
|
||||
if port:
|
||||
if port.is_output:
|
||||
self.editor.dragging_port = port
|
||||
return
|
||||
else:
|
||||
if self.editor.dragging_port is not None:
|
||||
self.editor._connect(self.editor.dragging_port, port)
|
||||
self.editor.dragging_port = None
|
||||
return
|
||||
|
||||
super().mousePressEvent(event)
|
||||
|
||||
|
||||
# NODE EDITOR
|
||||
class NodeEditor(QWidget):
|
||||
def __init__(self, module_catalog):
|
||||
super().__init__()
|
||||
self.setWindowTitle("Node Editor")
|
||||
|
||||
self.module_catalog = module_catalog
|
||||
|
||||
self.scene = QGraphicsScene()
|
||||
self.view = NodeGraphicsView(self.scene, self)
|
||||
|
||||
self.left = QListWidget()
|
||||
for name in module_catalog:
|
||||
QListWidgetItem(name, self.left)
|
||||
|
||||
self.log = QListWidget()
|
||||
|
||||
self.run_btn = QPushButton("Run")
|
||||
self.stop_btn = QPushButton("Stop")
|
||||
self.resume_btn = QPushButton("Resume")
|
||||
self.cancel_btn = QPushButton("Cancel")
|
||||
self.show_dag_btn = QPushButton("Show DAG")
|
||||
|
||||
self.add_btn = QPushButton("Add Node")
|
||||
|
||||
# Layout
|
||||
main_layout = QHBoxLayout(self)
|
||||
left_layout = QVBoxLayout()
|
||||
left_layout.addWidget(self.left)
|
||||
left_layout.addWidget(self.add_btn)
|
||||
left_layout.addWidget(self.run_btn)
|
||||
left_layout.addWidget(self.stop_btn)
|
||||
left_layout.addWidget(self.resume_btn)
|
||||
left_layout.addWidget(self.cancel_btn)
|
||||
left_layout.addWidget(self.show_dag_btn)
|
||||
left_layout.addWidget(self.log)
|
||||
|
||||
main_layout.addLayout(left_layout)
|
||||
main_layout.addWidget(self.view)
|
||||
|
||||
self.left.setMaximumWidth(220)
|
||||
main_layout.setStretch(1, 1)
|
||||
|
||||
# Events
|
||||
self.add_btn.clicked.connect(self.add_node)
|
||||
self.run_btn.clicked.connect(self.run_workflow)
|
||||
self.stop_btn.clicked.connect(self.stop_workflow)
|
||||
self.resume_btn.clicked.connect(self.resume_workflow)
|
||||
self.cancel_btn.clicked.connect(self.cancel_workflow)
|
||||
self.show_dag_btn.clicked.connect(self.show_dag)
|
||||
|
||||
self.nodes = []
|
||||
self.connections = []
|
||||
self.dragging_port = None
|
||||
|
||||
self.scheduler = None
|
||||
self.ctx = None
|
||||
self.execution_order = []
|
||||
self.current_index = 0
|
||||
self.workflow_running = False
|
||||
self.workflow_paused = False
|
||||
|
||||
# LOGGING
|
||||
def _log(self, text, color="black"):
|
||||
item = QListWidgetItem(text)
|
||||
item.setForeground(QColor(color))
|
||||
self.log.addItem(item)
|
||||
self.log.scrollToBottom()
|
||||
|
||||
# ADD NODE
|
||||
def add_node(self):
|
||||
sel = self.left.currentItem()
|
||||
if not sel:
|
||||
return
|
||||
|
||||
name = sel.text()
|
||||
module = self.module_catalog[name]()
|
||||
|
||||
node = NodeItem(module, name, self)
|
||||
node.setPos(len(self.nodes)*40, len(self.nodes)*40)
|
||||
self.scene.addItem(node)
|
||||
|
||||
self.nodes.append(node)
|
||||
self._log(f"Node added: {name}", "blue")
|
||||
|
||||
# FIND PORT
|
||||
def _find_port(self, item):
|
||||
while item:
|
||||
if isinstance(item, PortItem):
|
||||
return item
|
||||
item = item.parentItem()
|
||||
return None
|
||||
|
||||
# REMOVE CONNECTION
|
||||
def _remove_connection(self, conn):
|
||||
if conn in self.connections:
|
||||
self.scene.removeItem(conn)
|
||||
self.connections.remove(conn)
|
||||
self._log("Connection removed", "red")
|
||||
|
||||
# CONNECT WITH TYPE CHECK
|
||||
def _connect(self, out_port_item, in_port_item):
|
||||
out_port = out_port_item.parent_node.module.out_ports[out_port_item.index]
|
||||
in_port = in_port_item.parent_node.module.in_ports[in_port_item.index]
|
||||
|
||||
# Only output → input
|
||||
if not out_port_item.is_output or in_port_item.is_output:
|
||||
self._log("❌ Invalid direction", "red")
|
||||
return
|
||||
|
||||
# Only one connection per input
|
||||
for c in self.connections:
|
||||
if c.in_port is in_port_item:
|
||||
self._log("❌ Input already connected", "red")
|
||||
return
|
||||
|
||||
# Type checking
|
||||
try:
|
||||
out_port.connect(in_port)
|
||||
except Exception as e:
|
||||
self._log(f"❌ Type mismatch: {e}", "red")
|
||||
return
|
||||
|
||||
# Success
|
||||
conn = ConnectionItem(out_port_item, in_port_item, self)
|
||||
self.scene.addItem(conn)
|
||||
self.connections.append(conn)
|
||||
|
||||
self._log(
|
||||
f"✔ Connected: {out_port_item.parent_node.title}.{out_port.name} → "
|
||||
f"{in_port_item.parent_node.title}.{in_port.name}",
|
||||
"green"
|
||||
)
|
||||
|
||||
# UPDATE CONNECTIONS ON MOVE
|
||||
def _update_connections_for_node(self, node):
|
||||
for c in self.connections:
|
||||
if c.out_port.parent_node is node or c.in_port.parent_node is node:
|
||||
c.update_path()
|
||||
|
||||
# BUILD GRAPH
|
||||
def _build_graph(self):
|
||||
graph = {i: [] for i in range(len(self.nodes))}
|
||||
|
||||
for c in self.connections:
|
||||
out_idx = self.nodes.index(c.out_port.parent_node)
|
||||
in_idx = self.nodes.index(c.in_port.parent_node)
|
||||
graph[out_idx].append(in_idx)
|
||||
|
||||
return graph
|
||||
|
||||
# TOPOLOGICAL SORT
|
||||
def _topological_sort(self, graph):
|
||||
indeg = {k: 0 for k in graph}
|
||||
for u in graph:
|
||||
for v in graph[u]:
|
||||
indeg[v] += 1
|
||||
|
||||
q = [u for u in graph if indeg[u] == 0]
|
||||
order = []
|
||||
|
||||
while q:
|
||||
u = q.pop(0)
|
||||
order.append(u)
|
||||
for v in graph[u]:
|
||||
indeg[v] -= 1
|
||||
if indeg[v] == 0:
|
||||
q.append(v)
|
||||
|
||||
if len(order) != len(graph):
|
||||
return None
|
||||
|
||||
return order
|
||||
|
||||
# SHOW DAG
|
||||
def show_dag(self):
|
||||
graph = self._build_graph()
|
||||
|
||||
text = ""
|
||||
for src, dsts in graph.items():
|
||||
src_name = self.nodes[src].title
|
||||
if dsts:
|
||||
for d in dsts:
|
||||
text += f"{src_name} → {self.nodes[d].title}\n"
|
||||
else:
|
||||
text += f"{src_name} → (no outputs)\n"
|
||||
|
||||
dlg = QDialog(self)
|
||||
dlg.setWindowTitle("DAG Structure")
|
||||
layout = QVBoxLayout(dlg)
|
||||
txt = QTextEdit()
|
||||
txt.setReadOnly(True)
|
||||
txt.setText(text)
|
||||
layout.addWidget(txt)
|
||||
dlg.resize(400, 300)
|
||||
dlg.exec()
|
||||
|
||||
# WORKFLOW EXECUTION
|
||||
def run_workflow(self):
|
||||
self._log("Workflow started", "green")
|
||||
|
||||
self.ctx = ExecutionContext("gui_run")
|
||||
self.scheduler = Scheduler()
|
||||
|
||||
# Build workflow
|
||||
wf = Workflow("gui_workflow")
|
||||
for n in self.nodes:
|
||||
wf.add_child(n.module)
|
||||
|
||||
# Build graph
|
||||
graph = self._build_graph()
|
||||
order = self._topological_sort(graph)
|
||||
|
||||
if order is None:
|
||||
self._log("❌ Cycle detected in graph", "red")
|
||||
return
|
||||
|
||||
self.execution_order = order
|
||||
self.current_index = 0
|
||||
self.workflow_running = True
|
||||
self.workflow_paused = False
|
||||
|
||||
self._execute_next()
|
||||
|
||||
def stop_workflow(self):
|
||||
self.workflow_paused = True
|
||||
self._log("Paused", "orange")
|
||||
|
||||
def resume_workflow(self):
|
||||
self.workflow_paused = False
|
||||
self._log("Resumed", "green")
|
||||
self._execute_next()
|
||||
|
||||
def cancel_workflow(self):
|
||||
self.workflow_running = False
|
||||
self._log("Cancelled", "red")
|
||||
|
||||
# EXECUTE NEXT NODE
|
||||
def _execute_next(self):
|
||||
if not self.workflow_running or self.workflow_paused:
|
||||
return
|
||||
|
||||
if self.current_index >= len(self.execution_order):
|
||||
self._log("Workflow finished", "green")
|
||||
self.workflow_running = False
|
||||
return
|
||||
|
||||
idx = self.execution_order[self.current_index]
|
||||
node = self.nodes[idx]
|
||||
module = node.module
|
||||
|
||||
node.highlight(True)
|
||||
self._log(f"Running: {node.title}", "blue")
|
||||
|
||||
start = time.time()
|
||||
|
||||
try:
|
||||
self.scheduler.run(module, self.ctx)
|
||||
end = time.time()
|
||||
duration = round(end - start, 4)
|
||||
|
||||
self._log(f"Finished: {node.title} (time: {duration}s)", "green")
|
||||
self._show_node_output(module)
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"❌ Error in {node.title}: {e}", "red")
|
||||
|
||||
node.highlight(False)
|
||||
|
||||
self.current_index += 1
|
||||
self._execute_next()
|
||||
|
||||
# SHOW NODE OUTPUT (SUMMARY)
|
||||
def _show_node_output(self, module):
|
||||
self._log("Output:", "purple")
|
||||
|
||||
for out_port in module.out_ports:
|
||||
try:
|
||||
asset = self.ctx.get(out_port)
|
||||
data = asset.data
|
||||
|
||||
summary = self._summarize(data)
|
||||
self._log(f" {out_port.name}: {summary}", "black")
|
||||
|
||||
except Exception as e:
|
||||
self._log(f" {out_port.name}: <error reading output>", "red")
|
||||
|
||||
def _summarize(self, data):
|
||||
text = str(data)
|
||||
if len(text) > 200:
|
||||
return text[:200] + " ... (truncated)"
|
||||
return text
|
||||
@@ -0,0 +1,49 @@
|
||||
from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QFrame
|
||||
from PySide6.QtGui import QPainter, QColor, QPen
|
||||
from PySide6.QtCore import Qt, QRectF
|
||||
|
||||
class PortWidget(QFrame):
|
||||
def __init__(self, name: str, dtype_repr: str, is_output: bool = False, parent=None):
|
||||
super().__init__(parent)
|
||||
self.name = name
|
||||
self.dtype_repr = dtype_repr
|
||||
self.is_output = is_output
|
||||
self.setFixedSize(14, 14)
|
||||
self.setToolTip(f"{name}\n{dtype_repr}")
|
||||
self.color = QColor(34,139,34) if is_output else QColor(70,130,180)
|
||||
self.setStyleSheet("background:transparent;")
|
||||
|
||||
def paintEvent(self, event):
|
||||
p = QPainter(self)
|
||||
p.setRenderHint(QPainter.Antialiasing)
|
||||
pen = QPen(Qt.black)
|
||||
p.setPen(pen)
|
||||
p.setBrush(self.color)
|
||||
r = QRectF(1,1,self.width()-2,self.height()-2)
|
||||
p.drawEllipse(r)
|
||||
|
||||
class NodeWidget(QFrame):
|
||||
def __init__(self, title: str, parent=None):
|
||||
super().__init__(parent)
|
||||
self.setFrameShape(QFrame.Box)
|
||||
self.setStyleSheet("background:#f8f8f8;")
|
||||
self.title = QLabel(title)
|
||||
self.title.setStyleSheet("font-weight:bold;")
|
||||
self.layout = QVBoxLayout(self)
|
||||
self.layout.setContentsMargins(6,6,6,6)
|
||||
self.layout.addWidget(self.title)
|
||||
self.in_ports = []
|
||||
self.out_ports = []
|
||||
self.status_label = QLabel("status: pending")
|
||||
self.layout.addWidget(self.status_label)
|
||||
|
||||
def add_in_port(self, port_widget: PortWidget):
|
||||
self.in_ports.append(port_widget)
|
||||
self.layout.addWidget(port_widget)
|
||||
|
||||
def add_out_port(self, port_widget: PortWidget):
|
||||
self.out_ports.append(port_widget)
|
||||
self.layout.addWidget(port_widget)
|
||||
|
||||
def set_status(self, status_text: str):
|
||||
self.status_label.setText(f"status: {status_text}")
|
||||
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
import pysera
|
||||
from api.module import Module
|
||||
from api.io_port import InPort, OutPort, NIFTIImageType, CSVTableType
|
||||
|
||||
|
||||
def factory():
|
||||
m = Module("PySERAExtractor")
|
||||
|
||||
in_image = InPort("image", NIFTIImageType())
|
||||
in_mask = InPort("mask", NIFTIImageType())
|
||||
|
||||
out_features = OutPort("features", CSVTableType(columns=["name", "value"]))
|
||||
|
||||
m.add_in_port(in_image)
|
||||
m.add_in_port(in_mask)
|
||||
m.add_out_port(out_features)
|
||||
|
||||
def execute(inputs):
|
||||
image_input = inputs["image"]
|
||||
mask_input = inputs["mask"]
|
||||
|
||||
output_dir = "results"
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
result = pysera.process_batch(
|
||||
image_input=image_input,
|
||||
mask_input=mask_input,
|
||||
output_path=output_dir,
|
||||
num_workers="auto",
|
||||
enable_parallelism=True,
|
||||
apply_preprocessing=True,
|
||||
categories="all",
|
||||
dimensions="1st,2_5d,3d",
|
||||
feature_value_mode="REAL_VALUE",
|
||||
extraction_mode="handcrafted_feature",
|
||||
report="info",
|
||||
)
|
||||
|
||||
df = result.get("features_extracted")
|
||||
|
||||
features = []
|
||||
if df is not None:
|
||||
for idx, row in df.iterrows():
|
||||
features.append({
|
||||
"name": row[0],
|
||||
"value": row[1]
|
||||
})
|
||||
|
||||
return {
|
||||
"features": features
|
||||
}
|
||||
|
||||
m.execute = execute
|
||||
return m
|
||||
@@ -0,0 +1,108 @@
|
||||
import os
|
||||
import numpy as np
|
||||
import nibabel as nib
|
||||
import pydicom
|
||||
import nrrd
|
||||
import cv2
|
||||
|
||||
from api.module import Module
|
||||
from api.io_port import OutPort, NIFTIImageType
|
||||
|
||||
|
||||
def load_nifti(path):
|
||||
nii = nib.load(path)
|
||||
return nii.get_fdata().astype(np.float32)
|
||||
|
||||
|
||||
def load_dicom(path):
|
||||
if os.path.isdir(path):
|
||||
files = sorted([
|
||||
os.path.join(path, f)
|
||||
for f in os.listdir(path)
|
||||
if not f.startswith(".")
|
||||
])
|
||||
slices = [pydicom.dcmread(f).pixel_array for f in files]
|
||||
return np.stack(slices).astype(np.float32)
|
||||
|
||||
ds = pydicom.dcmread(path)
|
||||
return ds.pixel_array.astype(np.float32)
|
||||
|
||||
|
||||
def load_nrrd(path):
|
||||
data, _ = nrrd.read(path)
|
||||
return data.astype(np.float32)
|
||||
|
||||
|
||||
def load_numpy(path):
|
||||
return np.load(path).astype(np.float32)
|
||||
|
||||
def load_image(path):
|
||||
img = cv2.imread(path, cv2.IMREAD_UNCHANGED)
|
||||
if img is None:
|
||||
raise ValueError(f"Cannot read image file: {path}")
|
||||
return img.astype(np.float32)
|
||||
|
||||
|
||||
def load_any(path):
|
||||
path = path.replace("\\", "/").lower()
|
||||
|
||||
if path.endswith((".nii", ".nii.gz")):
|
||||
return load_nifti(path)
|
||||
|
||||
if path.endswith(".nrrd"):
|
||||
return load_nrrd(path)
|
||||
|
||||
if path.endswith(".npy"):
|
||||
return load_numpy(path)
|
||||
|
||||
if path.endswith((".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff")):
|
||||
return load_image(path)
|
||||
|
||||
if os.path.isdir(path) or path.endswith(".dcm"):
|
||||
return load_dicom(path)
|
||||
|
||||
raise ValueError(f"Unsupported image format: {path}")
|
||||
|
||||
|
||||
def factory():
|
||||
m = Module("ImageReader")
|
||||
|
||||
out_image = OutPort("image", NIFTIImageType(metadata={"role": "image"}))
|
||||
out_mask = OutPort("mask", NIFTIImageType(metadata={"role": "mask"}))
|
||||
|
||||
m.add_out_port(out_image)
|
||||
m.add_out_port(out_mask)
|
||||
|
||||
def execute(inputs):
|
||||
image_dir = "data/images"
|
||||
mask_dir = "data/masks"
|
||||
|
||||
image_files = [f for f in os.listdir(image_dir) if not f.startswith(".")]
|
||||
mask_files = [f for f in os.listdir(mask_dir) if not f.startswith(".")]
|
||||
|
||||
if not image_files:
|
||||
raise ValueError("No image found in data/images")
|
||||
|
||||
if not mask_files:
|
||||
raise ValueError("No mask found in data/masks")
|
||||
|
||||
image_path = os.path.join(image_dir, image_files[0])
|
||||
mask_path = os.path.join(mask_dir, mask_files[0])
|
||||
|
||||
try:
|
||||
image = load_any(image_path)
|
||||
except Exception as e:
|
||||
raise ValueError(f"Cannot read image: {image_path} ({e})")
|
||||
|
||||
try:
|
||||
mask = load_any(mask_path)
|
||||
except Exception as e:
|
||||
raise ValueError(f"Cannot read mask: {mask_path} ({e})")
|
||||
|
||||
return {
|
||||
"image": image,
|
||||
"mask": mask
|
||||
}
|
||||
|
||||
m.execute = execute
|
||||
return m
|
||||
@@ -0,0 +1,43 @@
|
||||
import os
|
||||
import csv
|
||||
from datetime import datetime
|
||||
from api.module import Module
|
||||
from api.io_port import InPort, CSVTableType
|
||||
|
||||
|
||||
def factory():
|
||||
m = Module("CSVWriter")
|
||||
|
||||
inp = InPort("features", CSVTableType(columns=["name", "value"]))
|
||||
m.add_in_port(inp)
|
||||
|
||||
def execute(inputs):
|
||||
data = inputs["features"]
|
||||
|
||||
rows = []
|
||||
try:
|
||||
import pandas as pd
|
||||
if hasattr(data, "iterrows"):
|
||||
for _, row in data.iterrows():
|
||||
rows.append({"name": row[0], "value": row[1]})
|
||||
else:
|
||||
rows = list(data)
|
||||
except Exception:
|
||||
rows = list(data)
|
||||
|
||||
os.makedirs("results", exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
path = os.path.join("results", f"features_{ts}.csv")
|
||||
|
||||
with open(path, "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["name", "value"])
|
||||
for r in rows:
|
||||
w.writerow([r["name"], r["value"]])
|
||||
|
||||
return {
|
||||
"csv_path": path
|
||||
}
|
||||
|
||||
m.execute = execute
|
||||
return m
|
||||
Reference in New Issue
Block a user