commit 9bb57e4799feba46462b59bf915d5f4d0ba94415 Author: soorena62 Date: Sun Feb 8 04:38:10 2026 +0330 Add All Folders diff --git a/Dagster/API_R&D/README.md b/Dagster/API_R&D/README.md new file mode 100644 index 0000000..3d1ae11 --- /dev/null +++ b/Dagster/API_R&D/README.md @@ -0,0 +1,8 @@ +Description: +# This project was created to test API layer codes in depth, focusing on all the capabilities, features, and methods implemented in the codes, such as dynamic type checking and type detection at the moment of data entry, connecting output ports to input, etc. in a real environment. This project has a GUI and allows you to create a workflow, select the required nodes, run the workflow, connect nodes, and manually. + +Dependencies: +# pip install dagster pysera pandas pyside6 numpy + +Running: +# python -m gui.app \ No newline at end of file diff --git a/Dagster/API_R&D/api/__init__.py b/Dagster/API_R&D/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Dagster/API_R&D/api/assets.py b/Dagster/API_R&D/api/assets.py new file mode 100644 index 0000000..764474a --- /dev/null +++ b/Dagster/API_R&D/api/assets.py @@ -0,0 +1,45 @@ +from typing import Any, Dict, Optional +from copy import deepcopy +from api.io_port import DType + + +class Asset: + """ + Represents the output of a Task. + + Stores: + - data: actual payload + - dtype: semantic type (DType instance) + - metadata: optional dictionary + """ + + def __init__(self, data: Any, dtype: Optional[DType] = None, metadata: Optional[Dict[str, Any]] = None): + self.data = data + self.dtype = dtype # DO NOT deepcopy dtype + self.metadata = metadata or {} + + # Whether this asset should be preserved after workflow execution + self.preserved = True + + # Lifecycle + def dismiss(self): + """Mark asset as disposable.""" + self.preserved = False + + # Utility + def clone(self) -> "Asset": + """ + Create a deep copy of the asset. + dtype is NOT deepcopied because DType objects are not deepcopy-safe. + """ + return Asset( + data=deepcopy(self.data), + dtype=self.dtype, # keep reference + metadata=deepcopy(self.metadata) + ) + + # Representation + def __repr__(self): + # Avoid printing full dtype object (may contain nested structures) + dtype_name = self.dtype.__class__.__name__ if self.dtype else "None" + return f"" diff --git a/Dagster/API_R&D/api/execution_context.py b/Dagster/API_R&D/api/execution_context.py new file mode 100644 index 0000000..d66b8c3 --- /dev/null +++ b/Dagster/API_R&D/api/execution_context.py @@ -0,0 +1,82 @@ +from typing import Dict, Any +from api.assets import Asset +from api.io_port import InPort, OutPort + + +class ExecutionContext: + """ + Runtime data container for a workflow execution. + Stores produced assets for OutPorts and allows InPorts to retrieve them. + """ + + def __init__(self, execution_id: str | None = None): + self.execution_id = execution_id + + # Key: OutPort, Value: Asset + self._data: Dict[OutPort, Asset] = {} + + # Arbitrary metadata (optional) + self._metadata: Dict[str, Any] = {} + + # Store produced data + def put(self, output_port: OutPort, asset: Asset): + """ + Register an asset produced by an OutPort. + Also updates output_port.produced_asset. + """ + self._data[output_port] = asset + output_port.produced_asset = asset + + # Retrieve data + def get(self, port): + """ + Retrieve asset for: + - OutPort → return its asset + - InPort → return asset from its connected OutPort + """ + if isinstance(port, OutPort): + return self._data.get(port) + + if isinstance(port, InPort): + if port.connected_output is None: + return None + return self._data.get(port.connected_output) + + raise TypeError("ExecutionContext.get() expects InPort or OutPort") + + # Check if data exists + def has_data(self, port) -> bool: + if isinstance(port, OutPort): + return port in self._data + + if isinstance(port, InPort): + if port.connected_output is None: + return False + return port.connected_output in self._data + + return False + + # Metadata management + def set_meta(self, key: str, value: Any): + self._metadata[key] = value + + def get_meta(self, key: str, default=None): + return self._metadata.get(key, default) + + # Debug snapshot + def status_view(self): + """ + Lightweight snapshot of current execution state. + Safe for GUI display. + """ + return { + "execution_id": self.execution_id, + "assets": { + f"{port.parent_task.name}.{port.name}": { + "dtype": port.dtype.__class__.__name__, + "preserved": asset.preserved, + } + for port, asset in self._data.items() + }, + "metadata": self._metadata.copy(), + } diff --git a/Dagster/API_R&D/api/io_port.py b/Dagster/API_R&D/api/io_port.py new file mode 100644 index 0000000..f474a53 --- /dev/null +++ b/Dagster/API_R&D/api/io_port.py @@ -0,0 +1,249 @@ +from abc import ABC, abstractmethod +from typing import Optional, Dict, Any, List + + +# Exceptions +class CompatibilityException(Exception): + def __init__(self, reason: str): + super().__init__(reason) + self.reason = reason + + +# DType (Semantic Contract) +class DType(ABC): + """ + Pure semantic type. Does NOT contain names. + Names belong to Ports or CompositePart. + """ + def __init__(self, metadata: Optional[Dict[str, Any]] = None): + self.metadata = metadata or {} + self.conditions: Optional[Dict[str, Any]] = None + + @abstractmethod + def can_connect_to(self, other: "DType", conditions: Optional[Dict[str, Any]] = None): + pass + + def set_conditions(self, conditions: Dict[str, Any]): + self.conditions = conditions + + def __repr__(self): + return f"{self.__class__.__name__}(metadata={self.metadata})" + + +# Composite Types +class CompositePart: + """ + Represents one named component inside a CompositeType. + Example: + CompositePart("image", ImageType("nifti")) + """ + def __init__(self, name: str, dtype: DType): + self.name = name + self.dtype = dtype + + def __repr__(self): + return f"CompositePart(name={self.name}, dtype={self.dtype})" + + +class CompositeType(DType): + """ + CompositeType is a LIST of CompositePart. + Order matters. Names are preserved inside CompositePart. + """ + def __init__(self, parts: List[CompositePart], metadata=None): + super().__init__(metadata) + self.parts = parts + + def can_connect_to(self, other: "DType", conditions=None): + if not isinstance(other, CompositeType): + raise CompatibilityException("Expected CompositeType") + + if len(self.parts) != len(other.parts): + raise CompatibilityException("CompositeType length mismatch") + + for i in range(len(self.parts)): + p1 = self.parts[i] + p2 = other.parts[i] + + if p1.name != p2.name: + raise CompatibilityException( + f"CompositeType part name mismatch: {p1.name} vs {p2.name}" + ) + + p1.dtype.can_connect_to(p2.dtype, conditions) + + def __repr__(self): + return f"CompositeType(parts={self.parts})" + + +# MaskType: +class MaskType(DType): + def __init__(self, metadata=None): + super().__init__(metadata) + + def can_connect_to(self, other: DType, conditions=None): + if not isinstance(other, MaskType): + raise CompatibilityException("Target is not a MaskType") + + +# Image Types: +class ImageType(DType): + def __init__(self, modality: Optional[str] = None, metadata=None): + super().__init__(metadata) + self.modality = modality + + def can_connect_to(self, other: DType, conditions=None): + if not isinstance(other, ImageType): + raise CompatibilityException("Target is not an ImageType") + + if self.modality and other.modality and self.modality != other.modality: + raise CompatibilityException( + f"Image modality mismatch: {self.modality} vs {other.modality}" + ) + + +class NIFTIImageType(ImageType): + def __init__(self, metadata=None): + super().__init__(modality="nifti", metadata=metadata) + + +class DICOMImageType(ImageType): + def __init__(self, metadata=None): + super().__init__(modality="dicom", metadata=metadata) + + +# Table Types: +class TableType(DType): + def __init__(self, columns: List[str], metadata=None): + super().__init__(metadata) + self.columns = columns + + def can_connect_to(self, other: DType, conditions=None): + if not isinstance(other, TableType): + raise CompatibilityException("Target is not a TableType") + + if self.columns != other.columns: + raise CompatibilityException( + f"Table schema mismatch: {self.columns} vs {other.columns}" + ) + + +class CSVTableType(TableType): + def __init__(self, columns: List[str], delimiter: str = ",", encoding: str = "utf-8", metadata=None): + super().__init__(columns, metadata) + self.delimiter = delimiter + self.encoding = encoding + + def can_connect_to(self, other: DType, conditions=None): + if not isinstance(other, CSVTableType): + raise CompatibilityException("CSV tables can only connect to CSV tables") + + super().can_connect_to(other, conditions) + + if self.delimiter != other.delimiter: + raise CompatibilityException( + f"CSV delimiter mismatch: {self.delimiter} vs {other.delimiter}" + ) + + +# Text Types: +class TextType(DType): + def __init__(self, language: Optional[str] = None, encoding: Optional[str] = None, metadata=None): + super().__init__(metadata) + self.language = language + self.encoding = encoding + + def can_connect_to(self, other: DType, conditions=None): + if not isinstance(other, TextType): + raise CompatibilityException("Target is not TextType") + + if self.language and other.language and self.language != other.language: + raise CompatibilityException( + f"Language mismatch: {self.language} vs {other.language}" + ) + + if self.encoding and other.encoding and self.encoding != other.encoding: + raise CompatibilityException( + f"Encoding mismatch: {self.encoding} vs {other.encoding}" + ) + + +# Ports: +class Port(ABC): + def __init__(self, name: str, dtype: DType): + self.name = name + self.dtype = dtype + self.parent_task = None # Set by Workflow builder + + def full_name(self): + if self.parent_task: + return f"{self.parent_task.name}.{self.name}" + return self.name + + def __repr__(self): + return f"{self.__class__.__name__}({self.full_name()}, {self.dtype})" + + +# OutPort (fan-out supported): +class OutPort(Port): + def __init__(self, name: str, dtype: DType): + super().__init__(name, dtype) + self._connections: List["InPort"] = [] + self.produced_asset = None # Set by ExecutionContext + + @property + def connections(self): + return list(self._connections) + + def connect(self, in_port: "InPort", conditions: Optional[Dict[str, Any]] = None): + # Fan-out allowed → no limit on connections + in_port.can_connect_to(self, conditions) + + self._connections.append(in_port) + in_port._connected_output = self + + def disconnect(self, in_port: "InPort"): + if in_port in self._connections: + self._connections.remove(in_port) + in_port._connected_output = None + + def get_downstream_ports(self): + return list(self._connections) + + +# InPort: +class InPort(Port): + def __init__(self, name: str, dtype: DType, required: bool = True): + super().__init__(name, dtype) + self.required = required + self._connected_output: Optional[OutPort] = None + + @property + def connected_output(self): + return self._connected_output + + def is_connected(self): + return self._connected_output is not None + + def is_ready(self, context): + """ + True only if data exists in ExecutionContext. + """ + if not self._connected_output: + return False + return context.has_data(self) + + def can_connect_to(self, out_port: OutPort, conditions: Optional[Dict[str, Any]] = None): + self.dtype.can_connect_to(out_port.dtype, conditions) + + def disconnect(self): + if self._connected_output: + self._connected_output._connections.remove(self) + self._connected_output = None + + def get_upstream_port(self): + return self._connected_output + + def validate(self): + if self.required and not self.is_connected(): + raise CompatibilityException(f"InPort {self.full_name()} is required but not connected") diff --git a/Dagster/API_R&D/api/module.py b/Dagster/API_R&D/api/module.py new file mode 100644 index 0000000..a66578d --- /dev/null +++ b/Dagster/API_R&D/api/module.py @@ -0,0 +1,89 @@ +from api.task import Task, Status, TaskEvent +from api.assets import Asset +from api.execution_context import ExecutionContext +from api.io_port import CompatibilityException, InPort, OutPort + + +class Module(Task): + """ + Leaf node: atomic executable unit. + Executes using its input ports and produces assets on output ports. + """ + + def add_in_port(self, port: InPort): + port.parent_task = self + self.in_ports.append(port) + + def add_out_port(self, port: OutPort): + port.parent_task = self + self.out_ports.append(port) + + # Hooks + def before_run(self, context: ExecutionContext): + pass + + def after_run(self, context: ExecutionContext): + pass + + def on_error(self, error: Exception): + pass + + # Validation + def validate(self): + super().validate() + + # Main execution entry point + def run(self, context: ExecutionContext): + self.before_run(context) + + if not self.check_inputs_ready(context): + self._set_status(Status.PENDING) + return + + self._set_status(Status.RUNNING) + self._emit(TaskEvent.ON_START, {"module": self.name}) + + try: + # Gather input data + consumed_data = {} + for port in self.in_ports: + if not port.is_ready(context): + raise CompatibilityException( + f"Input {port.full_name()} is not ready" + ) + asset = context.get(port) + consumed_data[port.name] = asset.data + + # Execute module logic + result = self.execute(consumed_data) + + if not isinstance(result, dict): + raise ValueError(f"Module '{self.name}' must return dict") + + # Store outputs + for out_port in self.out_ports: + if out_port.name not in result: + raise ValueError( + f"Output '{out_port.name}' missing in module '{self.name}' result" + ) + + asset = Asset( + data=result[out_port.name], + dtype=out_port.dtype + ) + + context.put(out_port, asset) + + self._set_status(Status.COMPLETED) + self._emit(TaskEvent.ON_FINISH, {"module": self.name}) + self.after_run(context) + + except Exception as e: + self._set_status(Status.FAILED) + self.on_error(e) + self._emit(TaskEvent.ON_ERROR, {"error": str(e)}) + raise e + + # Default execute + def execute(self, inputs): + return inputs diff --git a/Dagster/API_R&D/api/scheduler.py b/Dagster/API_R&D/api/scheduler.py new file mode 100644 index 0000000..23f6a60 --- /dev/null +++ b/Dagster/API_R&D/api/scheduler.py @@ -0,0 +1,64 @@ +from typing import Set +from api.task import Task, Status +from api.execution_context import ExecutionContext + + +class Scheduler: + """ + DAG-based execution scheduler. + Decides WHEN a Task can run based on port readiness and orchestrates execution. + """ + + def __init__(self): + self._completed: Set[Task] = set() + self._failed: Set[Task] = set() + + def run(self, root: Task, context: ExecutionContext): + self._execute_task(root, context) + + def _execute_task(self, task: Task, context: ExecutionContext): + if task.status in (Status.COMPLETED, Status.FAILED, Status.STOPPED): + return + + if hasattr(task, "children"): + self._run_workflow(task, context) + return + + if not task.check_inputs_ready(context): + print(f"[Scheduler] Task '{task.name}' not ready, skipping for now.") + return + + try: + task._set_status(Status.RUNNING) + task.run(context) + task._set_status(Status.COMPLETED) + self._completed.add(task) + except Exception as e: + task._set_status(Status.FAILED) + self._failed.add(task) + print(f"[Scheduler] Task '{task.name}' failed: {e}") + raise + + def _run_workflow(self, workflow: Task, context: ExecutionContext): + workflow._set_status(Status.RUNNING) + + remaining = set(workflow.children) + + while remaining: + progress_made = False + + for child in list(remaining): + if child.check_inputs_ready(context): + self._execute_task(child, context) + remaining.remove(child) + progress_made = True + + if not progress_made: + raise RuntimeError( + f"Workflow '{workflow.name}' deadlock: unresolved dependencies" + ) + + if any(c.status == Status.FAILED for c in workflow.children): + workflow._set_status(Status.FAILED) + else: + workflow._set_status(Status.COMPLETED) diff --git a/Dagster/API_R&D/api/task.py b/Dagster/API_R&D/api/task.py new file mode 100644 index 0000000..7f0367d --- /dev/null +++ b/Dagster/API_R&D/api/task.py @@ -0,0 +1,78 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import List, Any, Optional + +from api.io_port import InPort, OutPort +from api.execution_context import ExecutionContext + + +class Status(Enum): + PENDING = "pending" + READY = "ready" + RUNNING = "running" + PAUSED = "paused" + STOPPED = "stopped" + COMPLETED = "completed" + FAILED = "failed" + + +class TaskEvent(Enum): + BEFORE_RUN = "before_run" + AFTER_RUN = "after_run" + ON_ERROR = "on_error" + STATUS_CHANGED = "status_changed" + ON_START = "on_start" + ON_FINISH = "on_finish" + + +class TaskEventListener: + def handle(self, event: TaskEvent, task: "Task", payload: Optional[Any] = None): + pass + + +class Task(ABC): + def __init__(self, name: str): + self.name = name + self.status = Status.PENDING + + self.in_ports: List[InPort] = [] + self.out_ports: List[OutPort] = [] + + self._listeners: List[TaskEventListener] = [] + self.parent_task: Optional["Task"] = None + + # Event System + def add_listener(self, listener: TaskEventListener): + self._listeners.append(listener) + + def _emit(self, event: TaskEvent, payload: Optional[Any] = None): + for listener in self._listeners: + listener.handle(event, self, payload) + + def _set_status(self, new_status: Status): + old_status = self.status + self.status = new_status + + if old_status != new_status: + self._emit(TaskEvent.STATUS_CHANGED, {"from": old_status, "to": new_status}) + + # Input Readiness Check + def check_inputs_ready(self, context: ExecutionContext) -> bool: + for port in self.in_ports: + if not port.is_ready(context): + return False + return True + + # Execution Wrapper + def run(self, context: ExecutionContext): + """ + Base Task.run() SHOULD NOT be used for Module. + Module overrides run() completely. + """ + raise NotImplementedError( + "Task.run() should not be called for Module. Use Module.run() instead." + ) + + @abstractmethod + def execute(self, context: ExecutionContext): + pass diff --git a/Dagster/API_R&D/api/workflow.py b/Dagster/API_R&D/api/workflow.py new file mode 100644 index 0000000..51abbbe --- /dev/null +++ b/Dagster/API_R&D/api/workflow.py @@ -0,0 +1,96 @@ +from typing import List +from api.task import Task, Status, TaskEvent +from api.execution_context import ExecutionContext +from api.io_port import InPort, CompatibilityException + + +class Workflow(Task): + """ + Composite node: contains children Tasks. + Executes tasks in dependency order (topological execution). + """ + + def __init__(self, name: str): + super().__init__(name) + self.children: List[Task] = [] + + def execute(self, context: ExecutionContext): + pass + + def add_child(self, task: Task): + task.parent_task = self + self.children.append(task) + + def remove_child(self, task: Task): + if task in self.children: + self.children.remove(task) + task.parent_task = None + + def get_children(self): + return list(self.children) + + def get_all_tasks(self): + tasks = [] + for child in self.children: + tasks.append(child) + if isinstance(child, Workflow): + tasks.extend(child.get_all_tasks()) + return tasks + + def validate(self): + super().validate() + for child in self.children: + child.validate() + + def _get_dependencies(self, task: Task) -> List[Task]: + deps = [] + for in_port in task.in_ports: + if isinstance(in_port, InPort) and in_port.connected_output: + upstream_task = in_port.connected_output.parent_task + if upstream_task and upstream_task != task: + deps.append(upstream_task) + return deps + + def run(self, context: ExecutionContext): + self._set_status(Status.RUNNING) + self._emit(TaskEvent.ON_START, {"workflow": self.name}) + + remaining = set(self.children) + completed = set() + + while remaining: + progress_made = False + + for task in list(remaining): + deps = self._get_dependencies(task) + + if all(dep in completed for dep in deps): + if task.check_inputs_ready(context): + try: + task.run(context) + completed.add(task) + remaining.remove(task) + progress_made = True + + except CompatibilityException as e: + task._set_status(Status.FAILED) + self._emit(TaskEvent.ON_ERROR, {"error": e.reason, "task": task.name}) + self._set_status(Status.FAILED) + return + + except Exception as e: + task._set_status(Status.FAILED) + self._emit(TaskEvent.ON_ERROR, {"error": str(e), "task": task.name}) + self._set_status(Status.FAILED) + return + + if not progress_made: + self._set_status(Status.FAILED) + self._emit(TaskEvent.ON_ERROR, {"error": "Dependency deadlock detected"}) + return + + if all(t.status == Status.COMPLETED for t in self.children): + self._set_status(Status.COMPLETED) + self._emit(TaskEvent.ON_FINISH, {"workflow": self.name}) + else: + self._set_status(Status.FAILED) diff --git a/Dagster/API_R&D/dagster_adapter/__init__.py b/Dagster/API_R&D/dagster_adapter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Dagster/API_R&D/dagster_adapter/dagster_pipeline.py b/Dagster/API_R&D/dagster_adapter/dagster_pipeline.py new file mode 100644 index 0000000..fdc4903 --- /dev/null +++ b/Dagster/API_R&D/dagster_adapter/dagster_pipeline.py @@ -0,0 +1,66 @@ +from dagster import op, job, In, Out +import pysera +from api.execution_context import ExecutionContext +from api.assets import Asset + + +@op(out={"image_input": Out(str), "mask_input": Out(str)}) +def image_reader_op(): + image_input = "data/sample/images" + mask_input = "data/sample/masks" + return {"image_input": image_input, "mask_input": mask_input} + + +@op(ins={"image_input": In(str), "mask_input": In(str)}, out={"features": Out(Asset)}) +def pysera_extractor_op(image_input, mask_input): + output_dir = "results" + + result = pysera.process_batch( + image_input=image_input, + mask_input=mask_input, + output_path=output_dir, + num_workers="auto", + enable_parallelism=True, + apply_preprocessing=True, + categories="all", + dimensions="1st,2_5d,3d", + feature_value_mode="REAL_VALUE", + extraction_mode="handcrafted_feature", + report="info", + ) + + df = result.get("features_extracted") + return {"features": Asset(df, "pysera_features_df")} + +@op(ins={"features": In(Asset)}, out={"csv_path": Out(Asset)}) +def csv_writer_op(features): + import os + import csv + from datetime import datetime + + df = features.data + rows = [] + if hasattr(df, "iterrows"): + for _, row in df.iterrows(): + rows.append({"name": row[0], "value": row[1]}) + + os.makedirs("results", exist_ok=True) + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + path = os.path.join("results", f"features_dagster_{ts}.csv") + + with open(path, "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["name", "value"]) + for r in rows: + w.writerow([r["name"], r["value"]]) + + return {"csv_path": Asset(path, "csv_path")} + +@job +def radiuma_pysera_job(): + image_mask = image_reader_op() + feats = pysera_extractor_op( + image_input=image_mask["image_input"], + mask_input=image_mask["mask_input"], + ) + csv_writer_op(features=feats["features"]) diff --git a/Dagster/API_R&D/data/images/CT_pitch.nii.gz b/Dagster/API_R&D/data/images/CT_pitch.nii.gz new file mode 100644 index 0000000..1a51792 Binary files /dev/null and b/Dagster/API_R&D/data/images/CT_pitch.nii.gz differ diff --git a/Dagster/API_R&D/data/masks/CT_pitch_mask.nii.gz b/Dagster/API_R&D/data/masks/CT_pitch_mask.nii.gz new file mode 100644 index 0000000..b7a2ff1 Binary files /dev/null and b/Dagster/API_R&D/data/masks/CT_pitch_mask.nii.gz differ diff --git a/Dagster/API_R&D/gui/__init__.py b/Dagster/API_R&D/gui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Dagster/API_R&D/gui/app.py b/Dagster/API_R&D/gui/app.py new file mode 100644 index 0000000..be88705 --- /dev/null +++ b/Dagster/API_R&D/gui/app.py @@ -0,0 +1,21 @@ +import sys +from PySide6.QtWidgets import QApplication +from gui.node_editor import NodeEditor +from modules.image_reader import factory as image_reader_factory +from modules.extractor import factory as pysera_factory +from modules.writer import factory as csv_factory + +def main(): + app = QApplication(sys.argv) + catalog = { + "ImageReader": image_reader_factory, + "PySERAExtractor": pysera_factory, + "CSVWriter": csv_factory + } + editor = NodeEditor(catalog) + editor.resize(1100, 720) + editor.show() + sys.exit(app.exec()) + +if __name__ == "__main__": + main() diff --git a/Dagster/API_R&D/gui/node_editor.py b/Dagster/API_R&D/gui/node_editor.py new file mode 100644 index 0000000..a76ed09 --- /dev/null +++ b/Dagster/API_R&D/gui/node_editor.py @@ -0,0 +1,460 @@ +import time +from PySide6.QtWidgets import ( + QWidget, QGraphicsView, QGraphicsScene, QListWidget, QListWidgetItem, + QPushButton, QHBoxLayout, QVBoxLayout, QDialog, QTextEdit +) +from PySide6.QtGui import ( + QPainter, QPen, QColor, QPainterPath, QTransform, QMouseEvent +) +from PySide6.QtCore import Qt, QRectF +from PySide6.QtWidgets import QGraphicsItem + +from api.execution_context import ExecutionContext +from api.workflow import Workflow +from api.scheduler import Scheduler + + +# PORT ITEM +class PortItem(QGraphicsItem): + R = 6 + + def __init__(self, parent_node, name, is_output, index): + super().__init__(parent_node) + self.parent_node = parent_node + self.name = name + self.is_output = is_output + self.index = index + self.setFlag(QGraphicsItem.ItemIsSelectable) + + def boundingRect(self): + return QRectF(-self.R, -self.R, 2*self.R, 2*self.R) + + def paint(self, painter, option, widget=None): + painter.setRenderHint(QPainter.Antialiasing) + color = QColor(34,139,34) if self.is_output else QColor(70,130,180) + painter.setPen(QPen(Qt.black, 1)) + painter.setBrush(color) + painter.drawEllipse(self.boundingRect()) + + +# CONNECTION ITEM +class ConnectionItem(QGraphicsItem): + def __init__(self, out_port, in_port, editor): + super().__init__() + self.out_port = out_port + self.in_port = in_port + self.editor = editor + self.setZValue(-1) + self._rect = QRectF() + self._path = QPainterPath() + self.setFlag(QGraphicsItem.ItemIsSelectable) + + def update_path(self): + p1 = self.out_port.scenePos() + p2 = self.in_port.scenePos() + + new_rect = QRectF(p1, p2).normalized().adjusted(-20, -20, 20, 20) + if new_rect != self._rect: + self.prepareGeometryChange() + self._rect = new_rect + + path = QPainterPath(p1) + dx = (p2.x() - p1.x()) * 0.5 + path.cubicTo(p1.x() + dx, p1.y(), p2.x() - dx, p2.y(), p2.x(), p2.y()) + self._path = path + + def boundingRect(self): + return self._rect + + def paint(self, painter, option, widget=None): + self.update_path() + painter.setRenderHint(QPainter.Antialiasing) + + if self.isSelected(): + painter.setPen(QPen(QColor(255, 0, 0), 3)) + else: + painter.setPen(QPen(QColor(50, 50, 50), 2)) + + painter.drawPath(self._path) + + def mousePressEvent(self, event): + if event.button() == Qt.LeftButton: + self.editor._remove_connection(self) + super().mousePressEvent(event) + + +# NODE ITEM +class NodeItem(QGraphicsItem): + W = 200 + H = 80 + + def __init__(self, module, title, editor): + super().__init__() + self.module = module + self.title = title + self.editor = editor + self.running = False + + self.in_ports = [] + self.out_ports = [] + + self.setFlags(QGraphicsItem.ItemIsMovable | QGraphicsItem.ItemIsSelectable) + self._build_ports() + + def highlight(self, active): + self.running = active + self.update() + + def _build_ports(self): + y = 30 + for i, p in enumerate(self.module.in_ports): + port = PortItem(self, p.name, False, i) + port.setPos(0, y) + self.in_ports.append(port) + y += 20 + + y = 30 + for i, p in enumerate(self.module.out_ports): + port = PortItem(self, p.name, True, i) + port.setPos(self.W, y) + self.out_ports.append(port) + y += 20 + + def boundingRect(self): + return QRectF(0, 0, self.W, self.H) + + def paint(self, painter, option, widget=None): + painter.setRenderHint(QPainter.Antialiasing) + + if self.running: + painter.setBrush(QColor("#c8f7c5")) + else: + painter.setBrush(QColor("#f0f0f0")) + + painter.setPen(QPen(Qt.black, 1)) + painter.drawRect(self.boundingRect()) + painter.drawText(10, 20, self.title) + + def itemChange(self, change, value): + if change == QGraphicsItem.ItemPositionHasChanged: + self.editor._update_connections_for_node(self) + return super().itemChange(change, value) + + +# CUSTOM VIEW +class NodeGraphicsView(QGraphicsView): + def __init__(self, scene, editor): + super().__init__(scene) + self.editor = editor + self.setRenderHint(QPainter.Antialiasing) + + def mousePressEvent(self, event: QMouseEvent): + pos = self.mapToScene(event.pos()) + item = self.scene().itemAt(pos, QTransform()) + + port = self.editor._find_port(item) + if port: + if port.is_output: + self.editor.dragging_port = port + return + else: + if self.editor.dragging_port is not None: + self.editor._connect(self.editor.dragging_port, port) + self.editor.dragging_port = None + return + + super().mousePressEvent(event) + + +# NODE EDITOR +class NodeEditor(QWidget): + def __init__(self, module_catalog): + super().__init__() + self.setWindowTitle("Node Editor") + + self.module_catalog = module_catalog + + self.scene = QGraphicsScene() + self.view = NodeGraphicsView(self.scene, self) + + self.left = QListWidget() + for name in module_catalog: + QListWidgetItem(name, self.left) + + self.log = QListWidget() + + self.run_btn = QPushButton("Run") + self.stop_btn = QPushButton("Stop") + self.resume_btn = QPushButton("Resume") + self.cancel_btn = QPushButton("Cancel") + self.show_dag_btn = QPushButton("Show DAG") + + self.add_btn = QPushButton("Add Node") + + # Layout + main_layout = QHBoxLayout(self) + left_layout = QVBoxLayout() + left_layout.addWidget(self.left) + left_layout.addWidget(self.add_btn) + left_layout.addWidget(self.run_btn) + left_layout.addWidget(self.stop_btn) + left_layout.addWidget(self.resume_btn) + left_layout.addWidget(self.cancel_btn) + left_layout.addWidget(self.show_dag_btn) + left_layout.addWidget(self.log) + + main_layout.addLayout(left_layout) + main_layout.addWidget(self.view) + + self.left.setMaximumWidth(220) + main_layout.setStretch(1, 1) + + # Events + self.add_btn.clicked.connect(self.add_node) + self.run_btn.clicked.connect(self.run_workflow) + self.stop_btn.clicked.connect(self.stop_workflow) + self.resume_btn.clicked.connect(self.resume_workflow) + self.cancel_btn.clicked.connect(self.cancel_workflow) + self.show_dag_btn.clicked.connect(self.show_dag) + + self.nodes = [] + self.connections = [] + self.dragging_port = None + + self.scheduler = None + self.ctx = None + self.execution_order = [] + self.current_index = 0 + self.workflow_running = False + self.workflow_paused = False + + # LOGGING + def _log(self, text, color="black"): + item = QListWidgetItem(text) + item.setForeground(QColor(color)) + self.log.addItem(item) + self.log.scrollToBottom() + + # ADD NODE + def add_node(self): + sel = self.left.currentItem() + if not sel: + return + + name = sel.text() + module = self.module_catalog[name]() + + node = NodeItem(module, name, self) + node.setPos(len(self.nodes)*40, len(self.nodes)*40) + self.scene.addItem(node) + + self.nodes.append(node) + self._log(f"Node added: {name}", "blue") + + # FIND PORT + def _find_port(self, item): + while item: + if isinstance(item, PortItem): + return item + item = item.parentItem() + return None + + # REMOVE CONNECTION + def _remove_connection(self, conn): + if conn in self.connections: + self.scene.removeItem(conn) + self.connections.remove(conn) + self._log("Connection removed", "red") + + # CONNECT WITH TYPE CHECK + def _connect(self, out_port_item, in_port_item): + out_port = out_port_item.parent_node.module.out_ports[out_port_item.index] + in_port = in_port_item.parent_node.module.in_ports[in_port_item.index] + + # Only output → input + if not out_port_item.is_output or in_port_item.is_output: + self._log("❌ Invalid direction", "red") + return + + # Only one connection per input + for c in self.connections: + if c.in_port is in_port_item: + self._log("❌ Input already connected", "red") + return + + # Type checking + try: + out_port.connect(in_port) + except Exception as e: + self._log(f"❌ Type mismatch: {e}", "red") + return + + # Success + conn = ConnectionItem(out_port_item, in_port_item, self) + self.scene.addItem(conn) + self.connections.append(conn) + + self._log( + f"✔ Connected: {out_port_item.parent_node.title}.{out_port.name} → " + f"{in_port_item.parent_node.title}.{in_port.name}", + "green" + ) + + # UPDATE CONNECTIONS ON MOVE + def _update_connections_for_node(self, node): + for c in self.connections: + if c.out_port.parent_node is node or c.in_port.parent_node is node: + c.update_path() + + # BUILD GRAPH + def _build_graph(self): + graph = {i: [] for i in range(len(self.nodes))} + + for c in self.connections: + out_idx = self.nodes.index(c.out_port.parent_node) + in_idx = self.nodes.index(c.in_port.parent_node) + graph[out_idx].append(in_idx) + + return graph + + # TOPOLOGICAL SORT + def _topological_sort(self, graph): + indeg = {k: 0 for k in graph} + for u in graph: + for v in graph[u]: + indeg[v] += 1 + + q = [u for u in graph if indeg[u] == 0] + order = [] + + while q: + u = q.pop(0) + order.append(u) + for v in graph[u]: + indeg[v] -= 1 + if indeg[v] == 0: + q.append(v) + + if len(order) != len(graph): + return None + + return order + + # SHOW DAG + def show_dag(self): + graph = self._build_graph() + + text = "" + for src, dsts in graph.items(): + src_name = self.nodes[src].title + if dsts: + for d in dsts: + text += f"{src_name} → {self.nodes[d].title}\n" + else: + text += f"{src_name} → (no outputs)\n" + + dlg = QDialog(self) + dlg.setWindowTitle("DAG Structure") + layout = QVBoxLayout(dlg) + txt = QTextEdit() + txt.setReadOnly(True) + txt.setText(text) + layout.addWidget(txt) + dlg.resize(400, 300) + dlg.exec() + + # WORKFLOW EXECUTION + def run_workflow(self): + self._log("Workflow started", "green") + + self.ctx = ExecutionContext("gui_run") + self.scheduler = Scheduler() + + # Build workflow + wf = Workflow("gui_workflow") + for n in self.nodes: + wf.add_child(n.module) + + # Build graph + graph = self._build_graph() + order = self._topological_sort(graph) + + if order is None: + self._log("❌ Cycle detected in graph", "red") + return + + self.execution_order = order + self.current_index = 0 + self.workflow_running = True + self.workflow_paused = False + + self._execute_next() + + def stop_workflow(self): + self.workflow_paused = True + self._log("Paused", "orange") + + def resume_workflow(self): + self.workflow_paused = False + self._log("Resumed", "green") + self._execute_next() + + def cancel_workflow(self): + self.workflow_running = False + self._log("Cancelled", "red") + + # EXECUTE NEXT NODE + def _execute_next(self): + if not self.workflow_running or self.workflow_paused: + return + + if self.current_index >= len(self.execution_order): + self._log("Workflow finished", "green") + self.workflow_running = False + return + + idx = self.execution_order[self.current_index] + node = self.nodes[idx] + module = node.module + + node.highlight(True) + self._log(f"Running: {node.title}", "blue") + + start = time.time() + + try: + self.scheduler.run(module, self.ctx) + end = time.time() + duration = round(end - start, 4) + + self._log(f"Finished: {node.title} (time: {duration}s)", "green") + self._show_node_output(module) + + except Exception as e: + self._log(f"❌ Error in {node.title}: {e}", "red") + + node.highlight(False) + + self.current_index += 1 + self._execute_next() + + # SHOW NODE OUTPUT (SUMMARY) + def _show_node_output(self, module): + self._log("Output:", "purple") + + for out_port in module.out_ports: + try: + asset = self.ctx.get(out_port) + data = asset.data + + summary = self._summarize(data) + self._log(f" {out_port.name}: {summary}", "black") + + except Exception as e: + self._log(f" {out_port.name}: ", "red") + + def _summarize(self, data): + text = str(data) + if len(text) > 200: + return text[:200] + " ... (truncated)" + return text diff --git a/Dagster/API_R&D/gui/widgets.py b/Dagster/API_R&D/gui/widgets.py new file mode 100644 index 0000000..821f24d --- /dev/null +++ b/Dagster/API_R&D/gui/widgets.py @@ -0,0 +1,49 @@ +from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QFrame +from PySide6.QtGui import QPainter, QColor, QPen +from PySide6.QtCore import Qt, QRectF + +class PortWidget(QFrame): + def __init__(self, name: str, dtype_repr: str, is_output: bool = False, parent=None): + super().__init__(parent) + self.name = name + self.dtype_repr = dtype_repr + self.is_output = is_output + self.setFixedSize(14, 14) + self.setToolTip(f"{name}\n{dtype_repr}") + self.color = QColor(34,139,34) if is_output else QColor(70,130,180) + self.setStyleSheet("background:transparent;") + + def paintEvent(self, event): + p = QPainter(self) + p.setRenderHint(QPainter.Antialiasing) + pen = QPen(Qt.black) + p.setPen(pen) + p.setBrush(self.color) + r = QRectF(1,1,self.width()-2,self.height()-2) + p.drawEllipse(r) + +class NodeWidget(QFrame): + def __init__(self, title: str, parent=None): + super().__init__(parent) + self.setFrameShape(QFrame.Box) + self.setStyleSheet("background:#f8f8f8;") + self.title = QLabel(title) + self.title.setStyleSheet("font-weight:bold;") + self.layout = QVBoxLayout(self) + self.layout.setContentsMargins(6,6,6,6) + self.layout.addWidget(self.title) + self.in_ports = [] + self.out_ports = [] + self.status_label = QLabel("status: pending") + self.layout.addWidget(self.status_label) + + def add_in_port(self, port_widget: PortWidget): + self.in_ports.append(port_widget) + self.layout.addWidget(port_widget) + + def add_out_port(self, port_widget: PortWidget): + self.out_ports.append(port_widget) + self.layout.addWidget(port_widget) + + def set_status(self, status_text: str): + self.status_label.setText(f"status: {status_text}") \ No newline at end of file diff --git a/Dagster/API_R&D/modules/extractor.py b/Dagster/API_R&D/modules/extractor.py new file mode 100644 index 0000000..a746dce --- /dev/null +++ b/Dagster/API_R&D/modules/extractor.py @@ -0,0 +1,55 @@ +import os +import pysera +from api.module import Module +from api.io_port import InPort, OutPort, NIFTIImageType, CSVTableType + + +def factory(): + m = Module("PySERAExtractor") + + in_image = InPort("image", NIFTIImageType()) + in_mask = InPort("mask", NIFTIImageType()) + + out_features = OutPort("features", CSVTableType(columns=["name", "value"])) + + m.add_in_port(in_image) + m.add_in_port(in_mask) + m.add_out_port(out_features) + + def execute(inputs): + image_input = inputs["image"] + mask_input = inputs["mask"] + + output_dir = "results" + os.makedirs(output_dir, exist_ok=True) + + result = pysera.process_batch( + image_input=image_input, + mask_input=mask_input, + output_path=output_dir, + num_workers="auto", + enable_parallelism=True, + apply_preprocessing=True, + categories="all", + dimensions="1st,2_5d,3d", + feature_value_mode="REAL_VALUE", + extraction_mode="handcrafted_feature", + report="info", + ) + + df = result.get("features_extracted") + + features = [] + if df is not None: + for idx, row in df.iterrows(): + features.append({ + "name": row[0], + "value": row[1] + }) + + return { + "features": features + } + + m.execute = execute + return m diff --git a/Dagster/API_R&D/modules/image_reader.py b/Dagster/API_R&D/modules/image_reader.py new file mode 100644 index 0000000..5faa88e --- /dev/null +++ b/Dagster/API_R&D/modules/image_reader.py @@ -0,0 +1,108 @@ +import os +import numpy as np +import nibabel as nib +import pydicom +import nrrd +import cv2 + +from api.module import Module +from api.io_port import OutPort, NIFTIImageType + + +def load_nifti(path): + nii = nib.load(path) + return nii.get_fdata().astype(np.float32) + + +def load_dicom(path): + if os.path.isdir(path): + files = sorted([ + os.path.join(path, f) + for f in os.listdir(path) + if not f.startswith(".") + ]) + slices = [pydicom.dcmread(f).pixel_array for f in files] + return np.stack(slices).astype(np.float32) + + ds = pydicom.dcmread(path) + return ds.pixel_array.astype(np.float32) + + +def load_nrrd(path): + data, _ = nrrd.read(path) + return data.astype(np.float32) + + +def load_numpy(path): + return np.load(path).astype(np.float32) + +def load_image(path): + img = cv2.imread(path, cv2.IMREAD_UNCHANGED) + if img is None: + raise ValueError(f"Cannot read image file: {path}") + return img.astype(np.float32) + + +def load_any(path): + path = path.replace("\\", "/").lower() + + if path.endswith((".nii", ".nii.gz")): + return load_nifti(path) + + if path.endswith(".nrrd"): + return load_nrrd(path) + + if path.endswith(".npy"): + return load_numpy(path) + + if path.endswith((".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff")): + return load_image(path) + + if os.path.isdir(path) or path.endswith(".dcm"): + return load_dicom(path) + + raise ValueError(f"Unsupported image format: {path}") + + +def factory(): + m = Module("ImageReader") + + out_image = OutPort("image", NIFTIImageType(metadata={"role": "image"})) + out_mask = OutPort("mask", NIFTIImageType(metadata={"role": "mask"})) + + m.add_out_port(out_image) + m.add_out_port(out_mask) + + def execute(inputs): + image_dir = "data/images" + mask_dir = "data/masks" + + image_files = [f for f in os.listdir(image_dir) if not f.startswith(".")] + mask_files = [f for f in os.listdir(mask_dir) if not f.startswith(".")] + + if not image_files: + raise ValueError("No image found in data/images") + + if not mask_files: + raise ValueError("No mask found in data/masks") + + image_path = os.path.join(image_dir, image_files[0]) + mask_path = os.path.join(mask_dir, mask_files[0]) + + try: + image = load_any(image_path) + except Exception as e: + raise ValueError(f"Cannot read image: {image_path} ({e})") + + try: + mask = load_any(mask_path) + except Exception as e: + raise ValueError(f"Cannot read mask: {mask_path} ({e})") + + return { + "image": image, + "mask": mask + } + + m.execute = execute + return m diff --git a/Dagster/API_R&D/modules/writer.py b/Dagster/API_R&D/modules/writer.py new file mode 100644 index 0000000..d2cd4f8 --- /dev/null +++ b/Dagster/API_R&D/modules/writer.py @@ -0,0 +1,43 @@ +import os +import csv +from datetime import datetime +from api.module import Module +from api.io_port import InPort, CSVTableType + + +def factory(): + m = Module("CSVWriter") + + inp = InPort("features", CSVTableType(columns=["name", "value"])) + m.add_in_port(inp) + + def execute(inputs): + data = inputs["features"] + + rows = [] + try: + import pandas as pd + if hasattr(data, "iterrows"): + for _, row in data.iterrows(): + rows.append({"name": row[0], "value": row[1]}) + else: + rows = list(data) + except Exception: + rows = list(data) + + os.makedirs("results", exist_ok=True) + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + path = os.path.join("results", f"features_{ts}.csv") + + with open(path, "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["name", "value"]) + for r in rows: + w.writerow([r["name"], r["value"]]) + + return { + "csv_path": path + } + + m.execute = execute + return m diff --git a/Dagster/Check_Matching/core/__init__.py b/Dagster/Check_Matching/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Dagster/Check_Matching/core/compatibility.py b/Dagster/Check_Matching/core/compatibility.py new file mode 100644 index 0000000..f6c2f61 --- /dev/null +++ b/Dagster/Check_Matching/core/compatibility.py @@ -0,0 +1,13 @@ +from core.contracts import RequirementContract, DataContract + + +def check_compatibility( + provider: DataContract, + requirement: RequirementContract +): + # Central compatibility entry point + try: + requirement.is_satisfied_by(provider) + return True, "Compatible" + except ValueError as e: + return False, str(e) diff --git a/Dagster/Check_Matching/core/compatibility_result.py b/Dagster/Check_Matching/core/compatibility_result.py new file mode 100644 index 0000000..6d0cc02 --- /dev/null +++ b/Dagster/Check_Matching/core/compatibility_result.py @@ -0,0 +1,14 @@ +from enum import Enum +from pydantic import BaseModel + + +class CompatibilityLevel(str, Enum): + OK = "OK" + WARNING = "WARNING" + ERROR = "ERROR" + + +class CompatibilityResult(BaseModel): + level: CompatibilityLevel + message: str + suggestion: str | None = None diff --git a/Dagster/Check_Matching/core/contracts.py b/Dagster/Check_Matching/core/contracts.py new file mode 100644 index 0000000..7024fd0 --- /dev/null +++ b/Dagster/Check_Matching/core/contracts.py @@ -0,0 +1,36 @@ +from pydantic import BaseModel +from typing import Literal, Set + + +class DataContract(BaseModel): + # Describes what a node produces + data_type: str + provides: Set[str] = set() + + +class ImageDataContract(DataContract): + data_type: Literal["image"] = "image" + modality: Literal["CT", "MR"] + dim: Literal["2D", "3D"] + has_mask: bool = False + provides: Set[str] = {"image"} + + +class MaskDataContract(DataContract): + data_type: Literal["mask"] = "mask" + modality: Literal["CT", "MR"] + dim: Literal["2D", "3D"] + provides: Set[str] = {"mask"} + + +class FeatureTableContract(DataContract): + data_type: Literal["table"] = "table" + provides: Set[str] = {"features"} + + +class RequirementContract(BaseModel): + # Describes what a node requires + data_type: str + + def is_satisfied_by(self, provided: DataContract): + raise NotImplementedError diff --git a/Dagster/Check_Matching/core/contracts_radiomics.py b/Dagster/Check_Matching/core/contracts_radiomics.py new file mode 100644 index 0000000..566088e --- /dev/null +++ b/Dagster/Check_Matching/core/contracts_radiomics.py @@ -0,0 +1,21 @@ +from pydantic import BaseModel +from typing import Literal, Set, Optional + + +class ImageDataContract(BaseModel): + data_type: Literal["image"] = "image" + modality: Literal["CT", "PET", "MR"] + dim: Literal["2D", "3D", "2_5D"] + geometry_id: Optional[str] = None + has_mask: bool = False + + +class MaskDataContract(BaseModel): + data_type: Literal["mask"] = "mask" + modality: Literal["CT", "PET", "MR"] + dim: Literal["2D", "3D", "2_5D"] + geometry_id: Optional[str] = None + + +class RadiomicsFeatureTableContract(BaseModel): + data_type: Literal["table"] = "table" diff --git a/Dagster/Check_Matching/core/node.py b/Dagster/Check_Matching/core/node.py new file mode 100644 index 0000000..a2c3109 --- /dev/null +++ b/Dagster/Check_Matching/core/node.py @@ -0,0 +1,15 @@ +from typing import Dict +from core.contracts import DataContract, RequirementContract + + +class Node: + # Base class for all processing nodes + + name: str + inputs: Dict[str, RequirementContract] + outputs: Dict[str, DataContract] + + def __init__(self, name, inputs=None, outputs=None): + self.name = name + self.inputs = inputs or {} + self.outputs = outputs or {} diff --git a/Dagster/Check_Matching/core/workflow.py b/Dagster/Check_Matching/core/workflow.py new file mode 100644 index 0000000..3da5cf7 --- /dev/null +++ b/Dagster/Check_Matching/core/workflow.py @@ -0,0 +1,12 @@ +class Workflow: + def __init__(self): + self.connections = [] + + def connect(self, out_node, out_port, in_node, in_port): + self.connections.append((out_node, out_port, in_node, in_port)) + + def get_connection(self, in_node, in_port): + for o_node, o_port, i_node, i_port in self.connections: + if i_node == in_node and i_port == in_port: + return o_node, o_port + raise RuntimeError("Missing connection") diff --git a/Dagster/Check_Matching/core/workflow_executor.py b/Dagster/Check_Matching/core/workflow_executor.py new file mode 100644 index 0000000..610b07a --- /dev/null +++ b/Dagster/Check_Matching/core/workflow_executor.py @@ -0,0 +1,34 @@ +class WorkflowExecutor: + def __init__(self, graph): + self.graph = graph + self.cache = {} + + def execute_from_writer(self, writer_node): + # Execute dependencies first + inputs = {} + + for port_name, requirement in writer_node.inputs.items(): + src_node, src_port = self.graph.get_connection(writer_node, port_name) + inputs[port_name] = self._execute_node(src_node) + + writer_node.run(**inputs) + + def _execute_node(self, node): + if node in self.cache: + return self.cache[node] + + # Reader + if hasattr(node, "load_data"): + result = node.load_data() + self.cache[node] = result + return result + + # Processor + inputs = {} + for port_name in node.inputs: + src_node, _ = self.graph.get_connection(node, port_name) + inputs[port_name] = self._execute_node(src_node) + + result = node.run(**inputs) + self.cache[node] = result + return result diff --git a/Dagster/Check_Matching/demo_check.py b/Dagster/Check_Matching/demo_check.py new file mode 100644 index 0000000..7ff05a4 --- /dev/null +++ b/Dagster/Check_Matching/demo_check.py @@ -0,0 +1,19 @@ +from core.workflow import Graph +from nodes.image_reader import ImageReader +from nodes.feature_extractor import FeatureExtractor +from nodes.feature_writer import FeatureWriter + + +graph = Graph() + +reader = ImageReader(modality="CT") +extractor = FeatureExtractor() +writer = FeatureWriter() + +# ImageReader → FeatureExtractor +ok, msg = graph.can_connect(reader, "image", extractor, "image") +print("Reader → Extractor:", ok, msg) + +# FeatureExtractor → FeatureWriter +ok, msg = graph.can_connect(extractor, "features", writer, "features") +print("Extractor → Writer:", ok, msg) diff --git a/Dagster/Check_Matching/gui/__init__.py b/Dagster/Check_Matching/gui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Dagster/Check_Matching/gui/gui_main.py b/Dagster/Check_Matching/gui/gui_main.py new file mode 100644 index 0000000..62e56a9 --- /dev/null +++ b/Dagster/Check_Matching/gui/gui_main.py @@ -0,0 +1,14 @@ +import sys +from PySide6.QtWidgets import QApplication +from gui.visual_editor import VisualEditor + + +def main(): + app = QApplication(sys.argv) + window = VisualEditor() + window.show() + sys.exit(app.exec()) + + +if __name__ == "__main__": + main() diff --git a/Dagster/Check_Matching/gui/visual_editor.py b/Dagster/Check_Matching/gui/visual_editor.py new file mode 100644 index 0000000..14ede8f --- /dev/null +++ b/Dagster/Check_Matching/gui/visual_editor.py @@ -0,0 +1,164 @@ +import sys +from PySide6.QtWidgets import ( + QMainWindow, QWidget, QVBoxLayout, QTextEdit, + QPushButton, QLabel, QSplitter +) +from PySide6.QtCore import Qt +from PySide6 import QtGui +from NodeGraphQt import NodeGraph, BaseNode + +from core.workflow import Graph +from nodes.image_reader import ImageReader +from nodes.feature_extractor import FeatureExtractor +from nodes.feature_writer import FeatureWriter + + +# GUI node wrappers around core nodes + +class ImageReaderNode(BaseNode): + __identifier__ = "radiuma.gui" + NODE_NAME = "ImageReader" + + def __init__(self): + super().__init__() + self.add_output("image") + self.core_node = ImageReader() + + +class FeatureExtractorNode(BaseNode): + __identifier__ = "radiuma.gui" + NODE_NAME = "FeatureExtractor" + + def __init__(self): + super().__init__() + self.add_input("image") + self.add_output("features") + self.core_node = FeatureExtractor() + + +class FeatureWriterNode(BaseNode): + __identifier__ = "radiuma.gui" + NODE_NAME = "FeatureWriter" + + def __init__(self): + super().__init__() + self.add_input("features") + self.core_node = FeatureWriter() + + +class VisualEditor(QMainWindow): + # Visual workflow editor with dynamic semantic validation + + def __init__(self): + super().__init__() + self.setWindowTitle("Radiuma R&D – Semantic Workflow Editor") + self.resize(1200, 800) + + self.graph_engine = Graph() + self.graph = NodeGraph() + self.viewer = self.graph.widget + + self.graph.register_node(ImageReaderNode) + self.graph.register_node(FeatureExtractorNode) + self.graph.register_node(FeatureWriterNode) + + self.log_panel = QTextEdit() + self.log_panel.setReadOnly(True) + + btn_reader = QPushButton("Add ImageReader") + btn_extractor = QPushButton("Add FeatureExtractor") + btn_writer = QPushButton("Add FeatureWriter") + + btn_reader.clicked.connect( + lambda: self.graph.create_node("radiuma.gui.ImageReaderNode") + ) + btn_extractor.clicked.connect( + lambda: self.graph.create_node("radiuma.gui.FeatureExtractorNode") + ) + btn_writer.clicked.connect( + lambda: self.graph.create_node("radiuma.gui.FeatureWriterNode") + ) + + left_panel = QWidget() + left_layout = QVBoxLayout(left_panel) + left_layout.addWidget(QLabel("Nodes")) + left_layout.addWidget(btn_reader) + left_layout.addWidget(btn_extractor) + left_layout.addWidget(btn_writer) + left_layout.addStretch() + + top_splitter = QSplitter(Qt.Horizontal) + top_splitter.addWidget(left_panel) + top_splitter.addWidget(self.viewer) + + main_splitter = QSplitter(Qt.Vertical) + main_splitter.addWidget(top_splitter) + main_splitter.addWidget(self.log_panel) + main_splitter.setSizes([600, 200]) + + self.setCentralWidget(main_splitter) + + self.graph.port_connected.connect(self._on_port_connected) + self.graph.port_disconnected.connect(self._on_port_disconnected) + + def log(self, msg, level="INFO"): + self.log_panel.append(f"[{level}] {msg}") + + def _on_port_connected(self, port_a, port_b): + node_a = port_a.node() + node_b = port_b.node() + + core_a = getattr(node_a, "core_node", None) + core_b = getattr(node_b, "core_node", None) + + if not core_a or not core_b: + self._color_connection(port_a, port_b, (200, 200, 200)) + self.log("Missing core node mapping", "ERROR") + return + + # Normalize out → in + if port_a.type_() == "out": + out_node, out_port = core_a, port_a.name() + in_node, in_port = core_b, port_b.name() + gui_out, gui_in = port_a, port_b + else: + out_node, out_port = core_b, port_b.name() + in_node, in_port = core_a, port_a.name() + gui_out, gui_in = port_b, port_a + + ok, msg = self.graph_engine.can_connect( + out_node, out_port, in_node, in_port + ) + + if ok: + self._color_connection(gui_out, gui_in, (46, 204, 113)) + self.log(msg, "INFO") + else: + self._color_connection(gui_out, gui_in, (231, 76, 60)) + self.log(msg, "ERROR") + + def _on_port_disconnected(self, port_a, port_b): + self._color_connection(port_a, port_b, (180, 180, 180)) + self.log("Disconnected ports", "INFO") + + def _color_connection(self, port_a, port_b, color): + # Patch connection paint dynamically + scene = self.graph._viewer.scene() + for item in scene.items(): + if hasattr(item, "port1") and hasattr(item, "port2"): + if {item.port1, item.port2} == {port_a, port_b}: + setattr(item, "_custom_color", color) + if not hasattr(item, "_patched"): + original_paint = item.paint + + def patched_paint(painter, option, widget=None, + _orig=original_paint, _item=item): + pen = painter.pen() + pen.setColor(QtGui.QColor(*_item._custom_color)) + painter.setPen(pen) + _orig(painter, option, widget) + + item.paint = patched_paint + item._patched = True + item.update() + return diff --git a/Dagster/Check_Matching/nodes/__init__.py b/Dagster/Check_Matching/nodes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Dagster/Check_Matching/nodes/feature_extractor.py b/Dagster/Check_Matching/nodes/feature_extractor.py new file mode 100644 index 0000000..c3fc264 --- /dev/null +++ b/Dagster/Check_Matching/nodes/feature_extractor.py @@ -0,0 +1,45 @@ +from typing import Set +from core.node import Node +from core.contracts import ( + RequirementContract, + ImageDataContract, + FeatureTableContract +) + + +class ImageRequirement(RequirementContract): + data_type: str = "image" + modality: Set[str] + dim: Set[str] + requires_mask: bool = False + + def is_satisfied_by(self, provided: ImageDataContract): + # Validate semantic compatibility + if provided.data_type != self.data_type: + raise ValueError("Data type mismatch") + + if provided.modality not in self.modality: + raise ValueError("Modality mismatch") + + if provided.dim not in self.dim: + raise ValueError("Dimension mismatch") + + if self.requires_mask and not provided.has_mask: + raise ValueError("Mask is required but not provided") + + +class FeatureExtractor(Node): + def __init__(self): + super().__init__( + name="FeatureExtractor", + inputs={ + "image": ImageRequirement( + modality={"CT"}, + dim={"3D"}, + requires_mask=False + ) + }, + outputs={ + "features": FeatureTableContract() + } + ) diff --git a/Dagster/Check_Matching/nodes/feature_writer.py b/Dagster/Check_Matching/nodes/feature_writer.py new file mode 100644 index 0000000..98622e7 --- /dev/null +++ b/Dagster/Check_Matching/nodes/feature_writer.py @@ -0,0 +1,33 @@ +import json +from core.node import Node +from core.compatibility_result import CompatibilityResult, CompatibilityLevel + + +class FeatureWriteRequirement: + data_type: str = "table" + + def check(self, provided): + if provided.data_type != self.data_type: + return CompatibilityResult( + level=CompatibilityLevel.ERROR, + message="FeatureWriter requires feature table input" + ) + return CompatibilityResult( + level=CompatibilityLevel.OK, + message="Compatible" + ) + + +class FeatureWriter(Node): + def __init__(self, output_path: str): + self.output_path = output_path + + super().__init__( + name="FeatureWriter", + inputs={"features": FeatureWriteRequirement()}, + outputs={} + ) + + def run(self, features): + with open(self.output_path, "w", encoding="utf-8") as f: + json.dump(features, f, indent=2) diff --git a/Dagster/Check_Matching/nodes/image_reader.py b/Dagster/Check_Matching/nodes/image_reader.py new file mode 100644 index 0000000..78b5a00 --- /dev/null +++ b/Dagster/Check_Matching/nodes/image_reader.py @@ -0,0 +1,37 @@ +import nibabel as nib +from core.node import Node +from core.contracts_radiomics import ImageDataContract +from pathlib import Path +import uuid + + +class ImageReader(Node): + def __init__(self, path: str): + self.path = Path(path) + + # Read image header only + img = nib.load(str(self.path)) + header = img.header + + # Infer metadata + dim = "3D" if img.ndim == 3 else "2D" + modality = header.get("descrip", b"CT").decode(errors="ignore") or "CT" + + geometry_id = str(uuid.uuid4()) + + super().__init__( + name="ImageReader", + outputs={ + "image": ImageDataContract( + modality=modality, + dim=dim, + geometry_id=geometry_id, + has_mask=False + ) + } + ) + + def load_data(self): + # Load full image data only when execution starts + img = nib.load(str(self.path)) + return img.get_fdata() diff --git a/Dagster/Check_Matching/nodes/image_writer.py b/Dagster/Check_Matching/nodes/image_writer.py new file mode 100644 index 0000000..feae261 --- /dev/null +++ b/Dagster/Check_Matching/nodes/image_writer.py @@ -0,0 +1,35 @@ +import nibabel as nib +from core.node import Node +from core.compatibility_result import CompatibilityResult, CompatibilityLevel + + +class ImageWriteRequirement: + data_type: str = "image" + + def check(self, provided): + if provided.data_type != self.data_type: + return CompatibilityResult( + level=CompatibilityLevel.ERROR, + message="ImageWriter requires image input" + ) + return CompatibilityResult( + level=CompatibilityLevel.OK, + message="Compatible" + ) + + +class ImageWriter(Node): + def __init__(self, output_path: str): + self.output_path = output_path + + super().__init__( + name="ImageWriter", + inputs={"image": ImageWriteRequirement()}, + outputs={} + ) + + def run(self, image_array): + nib.save( + nib.Nifti1Image(image_array, None), + self.output_path + ) diff --git a/Dagster/Check_Matching/nodes/mask_reader.py b/Dagster/Check_Matching/nodes/mask_reader.py new file mode 100644 index 0000000..cc72045 --- /dev/null +++ b/Dagster/Check_Matching/nodes/mask_reader.py @@ -0,0 +1,27 @@ +import nibabel as nib +from core.node import Node +from core.contracts_radiomics import MaskDataContract +from pathlib import Path + + +class MaskReader(Node): + def __init__(self, path: str, geometry_id: str): + self.path = Path(path) + + img = nib.load(str(self.path)) + dim = "3D" if img.ndim == 3 else "2D" + + super().__init__( + name="MaskReader", + outputs={ + "mask": MaskDataContract( + modality="CT", + dim=dim, + geometry_id=geometry_id + ) + } + ) + + def load_data(self): + img = nib.load(str(self.path)) + return img.get_fdata() diff --git a/Dagster/Check_Matching/nodes/radiomics_feature_generator.py b/Dagster/Check_Matching/nodes/radiomics_feature_generator.py new file mode 100644 index 0000000..241552d --- /dev/null +++ b/Dagster/Check_Matching/nodes/radiomics_feature_generator.py @@ -0,0 +1,23 @@ +import pysera +from core.node import Node +from core.contracts_radiomics import RadiomicsFeatureTableContract + + +class RadiomicsFeatureGenerator(Node): + def __init__(self, output_path="./results"): + self.output_path = output_path + + super().__init__( + name="RadiomicsFeatureGenerator", + inputs={"image": None, "mask": None}, + outputs={"features": RadiomicsFeatureTableContract()} + ) + + def run(self, image_path, mask_path): + result = pysera.process_batch( + image_input=image_path, + mask_input=mask_path, + output_path=self.output_path, + report="info" + ) + return result diff --git a/Dagster/Dagster_Assets/README.md b/Dagster/Dagster_Assets/README.md new file mode 100644 index 0000000..8fddbfc --- /dev/null +++ b/Dagster/Dagster_Assets/README.md @@ -0,0 +1,22 @@ +# Dagster Assets Project + +This project demonstrates how to orchestrate a radiomics workflow using **Dagster Assets**. +Assets are declarative building blocks that represent data or computation results. Each asset is materialized in sequence to produce reproducible outputs. + +## Features +- Asset-based orchestration for radiomics extraction. +- Clear lineage: images → registration → fusion → filtering → mask alignment → feature extraction → final JSON/Excel. +- Custom IOManager to persist outputs as JSON files in the `artifacts/` directory. + +## Requirements +- Python 3.10+ +- Dagster +- SimpleITK +- Pandas +- PySERA + +Install dependencies: +```bash +pip install dagster simpleitk pandas pysera +# Running: +python run_pipeline.py diff --git a/Dagster/Dagster_Assets/dagster_runner.py b/Dagster/Dagster_Assets/dagster_runner.py new file mode 100644 index 0000000..f6938d9 --- /dev/null +++ b/Dagster/Dagster_Assets/dagster_runner.py @@ -0,0 +1,29 @@ +import sys +from dagster import materialize +import radiuma_assets + +# Ensure UTF-8 output for logs +sys.stdout.reconfigure(encoding="utf-8") + +def run_assets(): + print("Starting Dagster Assets run...") + + result = materialize( + [ + radiuma_assets.all_masks, + radiuma_assets.image_reader, + radiuma_assets.image_registration, + radiuma_assets.image_fusion, + radiuma_assets.image_conversion, + radiuma_assets.image_filter, + radiuma_assets.mask_registration, # align masks to filtered image geometry + radiuma_assets.feature_extraction, # consume filtered images + registered masks + radiuma_assets.image_write, + ], + resources={"io_manager": radiuma_assets.json_io_manager}, + ) + + print("Workflow completed successfully." if result.success else "Workflow failed.") + +if __name__ == "__main__": + run_assets() diff --git a/Dagster/Dagster_Assets/data/images/CT_pitch.nii.gz b/Dagster/Dagster_Assets/data/images/CT_pitch.nii.gz new file mode 100644 index 0000000..1a51792 Binary files /dev/null and b/Dagster/Dagster_Assets/data/images/CT_pitch.nii.gz differ diff --git a/Dagster/Dagster_Assets/data/masks/CT_pitch_mask.nii.gz b/Dagster/Dagster_Assets/data/masks/CT_pitch_mask.nii.gz new file mode 100644 index 0000000..b7a2ff1 Binary files /dev/null and b/Dagster/Dagster_Assets/data/masks/CT_pitch_mask.nii.gz differ diff --git a/Dagster/Dagster_Assets/radiuma_assets.py b/Dagster/Dagster_Assets/radiuma_assets.py new file mode 100644 index 0000000..423d91a --- /dev/null +++ b/Dagster/Dagster_Assets/radiuma_assets.py @@ -0,0 +1,302 @@ +import os +import pathlib +import time +import json +import sys +from typing import List, Dict + +import pandas as pd +import SimpleITK as sitk +from dagster import asset, IOManager, io_manager +import pysera + +# Ensure stdout can handle UTF-8 encoding +sys.stdout.reconfigure(encoding="utf-8") + +DATA_DIR = os.path.join("data", "images") +MASK_DIR = os.path.join("data", "masks") +ARTIFACTS_DIR = os.path.join("artifacts") +os.makedirs(ARTIFACTS_DIR, exist_ok=True) + + +# JSON-safe conversion helper +def convert_paths_and_dfs(obj): + # Convert nested objects to JSON-safe types + if isinstance(obj, dict): + return {k: convert_paths_and_dfs(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [convert_paths_and_dfs(v) for v in obj] + elif isinstance(obj, tuple): + return [convert_paths_and_dfs(v) for v in obj] + elif isinstance(obj, pathlib.Path): + return str(obj) + elif isinstance(obj, pd.DataFrame): + return obj.to_dict(orient="records") + else: + return obj + +# Custom IOManager: persist asset outputs as JSON in artifacts/ +class JsonFileIOManager(IOManager): + def handle_output(self, context, obj): + file_path = os.path.join(ARTIFACTS_DIR, f"{context.asset_key.path[-1]}.json") + safe_obj = convert_paths_and_dfs(obj) + with open(file_path, "w", encoding="utf-8") as f: + json.dump(safe_obj, f, indent=2, ensure_ascii=False) + context.log.info(f"Output written to {file_path}") + + def load_input(self, context): + file_path = os.path.join(ARTIFACTS_DIR, f"{context.upstream_output.asset_key.path[-1]}.json") + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + context.log.info(f"Input loaded from {file_path}") + return data + +@io_manager +def json_io_manager(_): + return JsonFileIOManager() + +# Masks discovery +@asset +def all_masks() -> List[str]: + files = [os.path.join(MASK_DIR, f) for f in os.listdir(MASK_DIR) if f.endswith(".nii.gz")] + if not files: + raise FileNotFoundError("No masks found in data/masks") + return files + +# Image reader (force float32 and persist) +@asset +def image_reader() -> List[str]: + reader_dir = os.path.join(ARTIFACTS_DIR, "reader") + os.makedirs(reader_dir, exist_ok=True) + + files = [os.path.join(DATA_DIR, f) for f in os.listdir(DATA_DIR) if f.endswith(".nii.gz")] + if not files: + raise FileNotFoundError("No images found in data/images") + + converted_paths = [] + for path in files: + img = sitk.ReadImage(path) + print(f"[reader] raw {os.path.basename(path)} -> dim={img.GetDimension()}, type={img.GetPixelIDTypeAsString()}") + img_float = sitk.Cast(img, sitk.sitkFloat32) + print(f"[reader] casted {os.path.basename(path)} -> type={img_float.GetPixelIDTypeAsString()}") + out_path = os.path.join(reader_dir, f"reader_{os.path.basename(path)}") + sitk.WriteImage(img_float, out_path) + converted_paths.append(out_path) + + return converted_paths + + +# Utilities for registration and I/O +def write_nifti(image: sitk.Image, out_path: str): + os.makedirs(os.path.dirname(out_path), exist_ok=True) + sitk.WriteImage(image, out_path) + +def cast_to_float32(img: sitk.Image, label: str) -> sitk.Image: + casted = sitk.Cast(img, sitk.sitkFloat32) + print(f"[registration] {label}: dim={casted.GetDimension()}, type={casted.GetPixelIDTypeAsString()}") + return casted + +def make_initial_transform(fixed: sitk.Image, moving: sitk.Image) -> sitk.Transform: + dim = fixed.GetDimension() + if dim == 2: + return sitk.CenteredTransformInitializer( + fixed, moving, sitk.Euler2DTransform(), sitk.CenteredTransformInitializerFilter.GEOMETRY + ) + elif dim == 3: + return sitk.CenteredTransformInitializer( + fixed, moving, sitk.VersorRigid3DTransform(), sitk.CenteredTransformInitializerFilter.GEOMETRY + ) + else: + raise RuntimeError(f"Unsupported image dimension: {dim}") + +# Image registration (robust casting + 2D/3D support) +@asset +def image_registration(image_reader: List[str]) -> List[str]: + fixed_raw = sitk.ReadImage(image_reader[0]) + print(f"[registration] fixed_raw: dim={fixed_raw.GetDimension()}, type={fixed_raw.GetPixelIDTypeAsString()}") + fixed = cast_to_float32(fixed_raw, "fixed_cast") + + R = sitk.ImageRegistrationMethod() + if fixed.GetDimension() == 2: + R.SetMetricAsMeanSquares() + R.SetInterpolator(sitk.sitkLinear) + else: + R.SetMetricAsMattesMutualInformation(numberOfHistogramBins=50) + R.SetMetricSamplingStrategy(R.RANDOM) + R.SetMetricSamplingPercentage(0.2) + R.SetInterpolator(sitk.sitkLinear) + + R.SetOptimizerAsRegularStepGradientDescent( + learningRate=2.0, minStep=1e-4, numberOfIterations=200, gradientMagnitudeTolerance=1e-8 + ) + R.SetOptimizerScalesFromPhysicalShift() + R.SetShrinkFactorsPerLevel(shrinkFactors=[4, 2, 1]) + R.SetSmoothingSigmasPerLevel(smoothingSigmas=[2, 1, 0]) + R.SmoothingSigmasAreSpecifiedInPhysicalUnitsOn() + + out_paths = [] + for img_path in image_reader: + moving_raw = sitk.ReadImage(img_path) + print(f"[registration] moving_raw: {os.path.basename(img_path)} dim={moving_raw.GetDimension()}, type={moving_raw.GetPixelIDTypeAsString()}") + moving = cast_to_float32(moving_raw, f"moving_cast:{os.path.basename(img_path)}") + + init_tx = make_initial_transform(fixed, moving) + R.SetInitialTransform(init_tx, inPlace=False) + + final_tx = R.Execute(fixed, moving) + registered = sitk.Resample(moving, fixed, final_tx, sitk.sitkLinear, 0.0, sitk.sitkFloat32) + + out_path = os.path.join(ARTIFACTS_DIR, f"registered_{os.path.basename(img_path)}") + write_nifti(registered, out_path) + out_paths.append(out_path) + + return out_paths + + +# Fusion (robust intensity normalization) +@asset +def image_fusion(image_registration: List[str]) -> List[str]: + fused_paths = [] + import numpy as np + for img_path in image_registration: + img = sitk.ReadImage(img_path) + arr = sitk.GetArrayFromImage(img) + p5, p95 = np.percentile(arr, [5, 95]) + arr = np.clip(arr, p5, p95) + arr = (arr - p5) / (p95 - p5) if p95 > p5 else arr * 0.0 + fused_img = sitk.GetImageFromArray(arr) + fused_img.CopyInformation(img) + out_path = os.path.join(ARTIFACTS_DIR, f"fused_{os.path.basename(img_path)}") + write_nifti(fused_img, out_path) + fused_paths.append(out_path) + return fused_paths + + +# Conversion (ensures consistent naming) +@asset +def image_conversion(image_fusion: List[str]) -> List[str]: + converted_paths = [] + for img_path in image_fusion: + img = sitk.ReadImage(img_path) + out_path = os.path.join(ARTIFACTS_DIR, f"converted_{os.path.basename(img_path)}") + write_nifti(img, out_path) + converted_paths.append(out_path) + return converted_paths + +# Filter (Gaussian smoothing) +@asset +def image_filter(image_conversion: List[str]) -> List[str]: + filtered_paths = [] + for img_path in image_conversion: + img = sitk.ReadImage(img_path) + filtered_img = sitk.SmoothingRecursiveGaussian(img, sigma=1.0) + out_path = os.path.join(ARTIFACTS_DIR, f"filtered_{os.path.basename(img_path)}") + write_nifti(filtered_img, out_path) + filtered_paths.append(out_path) + return filtered_paths + +# Mask registration (nearest neighbor to filtered image geometry) +@asset +def mask_registration(image_filter: List[str], all_masks: List[str]) -> List[str]: + if not image_filter or not all_masks: + raise FileNotFoundError("Missing filtered images or masks for mask_registration.") + + registered_mask_paths = [] + + # Pair masks to images if lengths match; otherwise, resample all masks to the first filtered image + if len(image_filter) == len(all_masks): + pairs = zip(image_filter, all_masks) + else: + ref_path = image_filter[0] + pairs = [(ref_path, m) for m in all_masks] + + for ref_img_path, mask_path in pairs: + ref_img = sitk.ReadImage(ref_img_path) + mask_img = sitk.ReadImage(mask_path) + + identity = sitk.Transform(ref_img.GetDimension(), sitk.sitkIdentity) + resampled_mask = sitk.Resample( + mask_img, ref_img, identity, sitk.sitkNearestNeighbor, 0, mask_img.GetPixelID() + ) + + out_path = os.path.join(ARTIFACTS_DIR, f"mask_registered_{os.path.basename(mask_path)}") + sitk.WriteImage(resampled_mask, out_path) + registered_mask_paths.append(out_path) + + return registered_mask_paths + +# Feature extraction (PySeRA, returns JSON-serializable summary) + +@asset +def feature_extraction(image_filter: List[str], mask_registration: List[str]) -> Dict[str, list]: + results = [] + for img, mask in zip(image_filter, mask_registration): + start = time.time() + + result = pysera.process_batch( + image_input=img, + mask_input=mask, + output_path=ARTIFACTS_DIR, + categories="diag,morph,glcm,glrlm,glszm,ngtdm,ngldm", + dimensions="1st,3D", + bin_size=25, + roi_num=2, + roi_selection_mode="per_region", + apply_preprocessing=True, + feature_value_mode="REAL_VALUE", + min_roi_volume=50, + enable_parallelism=True, + num_workers=4, + report="info", + temporary_files_path=r"C:\\Users\\Omen16\\AppData\\Local\\ViSERA\\res\\memory\\memmap\\pysera_temp", + IBSI_based_parameters={ + "radiomics_DataType": "CT", + "radiomics_DiscType": "FBS", + "radiomics_isScale": 0, + "radiomics_VoxInterp": "Nearest", + "radiomics_ROIInterp": "Nearest", + "radiomics_isotVoxSize": 2.0, + "radiomics_isotVoxSize2D": 2.0, + "radiomics_isIsot2D": 0, + "radiomics_isGLround": 0, + "radiomics_isReSegRng": 0, + "radiomics_isOutliers": 0, + "radiomics_isQuntzStat": 1, + "radiomics_ReSegIntrvl01": -1000, + "radiomics_ReSegIntrvl02": 400, + "radiomics_ROI_PV": 0.5, + "radiomics_qntz": "Uniform", + "radiomics_IVH_Type": 3, + "radiomics_IVH_DiscCont": 1, + "radiomics_IVH_binSize": 2.0, + }, + ) + + elapsed = round(time.time() - start, 2) + + # Persist detailed per-case result for inspection + safe_result = convert_paths_and_dfs(result) + case_json = os.path.join(ARTIFACTS_DIR, f"{os.path.basename(img)}_radiomics.json") + with open(case_json, "w", encoding="utf-8") as f: + json.dump(safe_result, f, indent=2, ensure_ascii=False) + + print(f"Radiomics for {os.path.basename(img)} completed in {elapsed:.2f} seconds") + + # Append summary record + results.append({ + "image": img, + "mask": mask, + "elapsed_seconds": elapsed, + "result_file": case_json, + }) + + return {"radiomics_results": results} + +# Final writer (summary JSON) + +@asset +def image_write(feature_extraction: Dict[str, list]) -> str: + out_path = os.path.join(ARTIFACTS_DIR, "final_output.json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(feature_extraction, f, indent=2, ensure_ascii=False) + return out_path diff --git a/Dagster/Dagster_Assets/requirements.txt b/Dagster/Dagster_Assets/requirements.txt new file mode 100644 index 0000000..ff5cd52 --- /dev/null +++ b/Dagster/Dagster_Assets/requirements.txt @@ -0,0 +1,78 @@ +alembic==1.17.2 +annotated-types==0.7.0 +antlr4-python3-runtime==4.13.2 +certifi==2025.11.12 +charset-normalizer==3.4.4 +click==8.3.1 +colorama==0.4.6 +coloredlogs==14.0 +connected-components-3d==3.26.1 +dagster==1.12.6 +dagster-pipes==1.12.6 +dagster_shared==1.12.6 +dataclasses==0.6 +docstring_parser==0.17.0 +et_xmlfile==2.0.0 +filelock==3.20.0 +fsspec==2025.12.0 +greenlet==3.3.0 +grpcio==1.76.0 +grpcio-health-checking==1.76.0 +humanfriendly==10.0 +idna==3.11 +ImageIO==2.37.2 +Jinja2==3.1.6 +joblib==1.5.2 +lazy_loader==0.4 +Mako==1.3.10 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +networkx==3.6.1 +nibabel==5.3.3 +numpy==2.2.6 +opencv-python==4.12.0.88 +openpyxl==3.1.5 +packaging==25.0 +pandas==2.3.3 +pathlib_abc==0.5.2 +pillow==12.0.0 +platformdirs==4.5.1 +protobuf==6.33.2 +psutil==7.1.3 +pydantic==2.12.5 +pydantic_core==2.41.5 +pydicom==3.0.1 +Pygments==2.19.2 +pynrrd==1.1.3 +pyreadline3==3.5.4 +pysera==2.1.5 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +pytz==2025.2 +pywin32==311 +PyYAML==6.0.3 +requests==2.32.5 +rich==14.2.0 +rt-utils==1.2.7 +scikit-image==0.25.2 +scikit-learn==1.8.0 +scipy==1.16.3 +setuptools==80.9.0 +simpleitk==2.5.3 +six==1.17.0 +SQLAlchemy==2.0.45 +structlog==25.5.0 +tabulate==0.9.0 +threadpoolctl==3.6.0 +tifffile==2025.12.12 +tomli==2.3.0 +tomlkit==0.13.3 +toposort==1.10 +tqdm==4.67.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2025.2 +universal_pathlib==0.3.7 +urllib3==2.6.2 +watchdog==6.0.0 diff --git a/Dagster/Dagster_Minimal_Mode/README.md b/Dagster/Dagster_Minimal_Mode/README.md new file mode 100644 index 0000000..49e7af5 --- /dev/null +++ b/Dagster/Dagster_Minimal_Mode/README.md @@ -0,0 +1,26 @@ +## README For **Dagster_Minimal_Mode** + +```markdown +# Dagster Minimal Mode Project + +This project demonstrates the radiomics workflow in **Dagster Minimal Mode**. +Minimal Mode is a lightweight configuration of Dagster, focusing on simplicity and reduced overhead. + +## Features +- Minimal orchestration setup for radiomics extraction. +- Direct execution of ops/assets without full Dagster deployment. +- Simplified configuration for quick testing and prototyping. + +## Requirements +- Python 3.10+ +- Dagster (minimal mode enabled) +- SimpleITK +- Pandas +- PySERA + +Install dependencies: +```bash +pip install dagster simpleitk pandas pysera + +# Running: +python radiuma_pipeline.py diff --git a/Dagster/Dagster_Minimal_Mode/data/images/CT_pitch.nii.gz b/Dagster/Dagster_Minimal_Mode/data/images/CT_pitch.nii.gz new file mode 100644 index 0000000..1a51792 Binary files /dev/null and b/Dagster/Dagster_Minimal_Mode/data/images/CT_pitch.nii.gz differ diff --git a/Dagster/Dagster_Minimal_Mode/data/masks/CT_pitch_mask.nii.gz b/Dagster/Dagster_Minimal_Mode/data/masks/CT_pitch_mask.nii.gz new file mode 100644 index 0000000..b7a2ff1 Binary files /dev/null and b/Dagster/Dagster_Minimal_Mode/data/masks/CT_pitch_mask.nii.gz differ diff --git a/Dagster/Dagster_Minimal_Mode/ops_radiomics.py b/Dagster/Dagster_Minimal_Mode/ops_radiomics.py new file mode 100644 index 0000000..235f163 --- /dev/null +++ b/Dagster/Dagster_Minimal_Mode/ops_radiomics.py @@ -0,0 +1,11 @@ +from dagster import op +import pysera + +@op +def extract_features(inputs): + result = pysera.process_batch( + image_input="data/images/ourT1.nii.gz", + mask_input="data/masks/ourT1_mask.nii.gz", + output_path="artifacts" + ) + return result diff --git a/Dagster/Dagster_Minimal_Mode/ops_reader.py b/Dagster/Dagster_Minimal_Mode/ops_reader.py new file mode 100644 index 0000000..febedfa --- /dev/null +++ b/Dagster/Dagster_Minimal_Mode/ops_reader.py @@ -0,0 +1,74 @@ +# ops_reader.py +from dagster import op +import os +import pysera + +@op +def read_images(): + # Match Radiuma.exe by processing a single pair or a flat folder. + # Here we keep folders; PySERA will find matching pairs. + image_dir = "data/images" + mask_dir = "data/masks" + return {"image_dir": image_dir, "mask_dir": mask_dir} + +@op +def extract_features(data): + # Exact PySERA config mirrored from Radiuma.exe logs + result = pysera.process_batch( + image_input=data["image_dir"], + mask_input=data["mask_dir"], + output_path="./artifacts/results", + # Core run behavior + enable_parallelism=False, + num_workers=1, + apply_preprocessing=True, + roi_selection_mode="per_region", + roi_num=2, + min_roi_volume=50, + feature_value_mode="REAL_VALUE", + # Feature scope + categories="diag,morph,glcm,glrlm,glszm,ngtdm,ngldm", + dimensions="1st,2D", + bin_size=25, + # Logging/report + report="info", + # Temp path (matching Radiuma.exe run) + temporary_files_path=r"C:\Users\Omen16\AppData\Local\ViSERA\res\memory\memmap\pysera_temp", + # IBSI-based parameters mirrored from Radiuma.exe + IBSI_based_parameters={ + "radiomics_DataType": "CT", + "radiomics_DiscType": "FBS", + "radiomics_isScale": 0, + "radiomics_VoxInterp": "Nearest", + "radiomics_ROIInterp": "Nearest", + "radiomics_isotVoxSize": 1.0, + "radiomics_isotVoxSize2D": 2.0, + "radiomics_isIsot2D": 0, + "radiomics_isGLround": 0, + "radiomics_isReSegRng": 0, + "radiomics_isOutliers": 0, + "radiomics_isQuntzStat": 1, + "radiomics_ReSegIntrvl01": -1000, + "radiomics_ReSegIntrvl02": 400, + "radiomics_ROI_PV": 0.5, + "radiomics_qntz": "Uniform", + "radiomics_IVH_Type": 3, + "radiomics_IVH_DiscCont": 1, + "radiomics_IVH_binSize": 2.0, + }, + ) + return result + +@op +def write_report(result): + os.makedirs("artifacts", exist_ok=True) + report_path = "artifacts/radiomics_batch_report_radiuma_match.txt" + + with open(report_path, "w") as f: + f.write(f"Success: {result['success']}\n") + f.write(f"Processed files: {result['processed_files']}\n") + f.write(f"Processing time: {result['processing_time']:.2f} seconds\n") + f.write(f"Output path: {result['output_path']}\n") + f.write("Note: Excel with Radiomics_Features, Parameters, Report is saved in output_path.\n") + + return report_path \ No newline at end of file diff --git a/Dagster/Dagster_Minimal_Mode/ops_writer.py b/Dagster/Dagster_Minimal_Mode/ops_writer.py new file mode 100644 index 0000000..9e7ffed --- /dev/null +++ b/Dagster/Dagster_Minimal_Mode/ops_writer.py @@ -0,0 +1,9 @@ +from dagster import op + +@op +def write_report(features): + with open("artifacts/final_report.txt", "w", encoding="utf-8") as f: + f.write("# Radiomics Report\n") + f.write(f"Extracted {len(features)} features\n") + for k, v in features.items(): + f.write(f"{k},{v}\n") diff --git a/Dagster/Dagster_Minimal_Mode/radiuma_pipeline.py b/Dagster/Dagster_Minimal_Mode/radiuma_pipeline.py new file mode 100644 index 0000000..91bf6d1 --- /dev/null +++ b/Dagster/Dagster_Minimal_Mode/radiuma_pipeline.py @@ -0,0 +1,8 @@ +# radiuma_pipeline.py +from dagster import job +from ops_reader import read_images, extract_features, write_report + +@job +def radiuma_job(): + result = extract_features(read_images()) + write_report(result) \ No newline at end of file diff --git a/Dagster/Dagster_Minimal_Mode/requirements.txt b/Dagster/Dagster_Minimal_Mode/requirements.txt new file mode 100644 index 0000000..1ba9679 --- /dev/null +++ b/Dagster/Dagster_Minimal_Mode/requirements.txt @@ -0,0 +1,97 @@ +alembic==1.17.2 +annotated-types==0.7.0 +antlr4-python3-runtime==4.13.2 +anyio==4.12.0 +backoff==2.2.1 +certifi==2025.11.12 +charset-normalizer==3.4.4 +click==8.3.1 +colorama==0.4.6 +coloredlogs==14.0 +connected-components-3d==3.26.1 +dagit==1.12.6 +dagster==1.12.6 +dagster-graphql==1.12.6 +dagster-pipes==1.12.6 +dagster-webserver==1.12.6 +dagster_shared==1.12.6 +dataclasses==0.6 +docstring_parser==0.17.0 +et_xmlfile==2.0.0 +filelock==3.20.0 +fsspec==2025.12.0 +gql==3.5.3 +graphene==3.4.3 +graphql-core==3.2.6 +graphql-relay==3.2.0 +greenlet==3.3.0 +grpcio==1.76.0 +grpcio-health-checking==1.76.0 +h11==0.16.0 +httptools==0.7.1 +humanfriendly==10.0 +idna==3.11 +ImageIO==2.37.2 +Jinja2==3.1.6 +joblib==1.5.2 +lazy_loader==0.4 +Mako==1.3.10 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +multidict==6.7.0 +networkx==3.6.1 +nibabel==5.3.3 +numpy==2.2.6 +opencv-python==4.12.0.88 +openpyxl==3.1.5 +packaging==25.0 +pandas==2.3.3 +pathlib_abc==0.5.2 +pillow==12.0.0 +platformdirs==4.5.1 +propcache==0.4.1 +protobuf==6.33.2 +psutil==7.1.3 +pydantic==2.12.5 +pydantic_core==2.41.5 +pydicom==3.0.1 +Pygments==2.19.2 +pynrrd==1.1.3 +pyreadline3==3.5.4 +pysera==2.1.5 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +pytz==2025.2 +pywin32==311 +PyYAML==6.0.3 +requests==2.32.5 +requests-toolbelt==1.0.0 +rich==14.2.0 +rt-utils==1.2.7 +scikit-image==0.25.2 +scikit-learn==1.8.0 +scipy==1.16.3 +setuptools==80.9.0 +simpleitk==2.5.3 +six==1.17.0 +SQLAlchemy==2.0.45 +starlette==0.50.0 +structlog==25.5.0 +tabulate==0.9.0 +threadpoolctl==3.6.0 +tifffile==2025.10.16 +tomli==2.3.0 +tomlkit==0.13.3 +toposort==1.10 +tqdm==4.67.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2025.2 +universal_pathlib==0.3.7 +urllib3==2.6.2 +uvicorn==0.38.0 +watchdog==6.0.0 +watchfiles==1.1.1 +websockets==15.0.1 +yarl==1.22.0 diff --git a/Dagster/Dagster_Op/README.md b/Dagster/Dagster_Op/README.md new file mode 100644 index 0000000..f170f39 --- /dev/null +++ b/Dagster/Dagster_Op/README.md @@ -0,0 +1,28 @@ +## README **Dagster_op** + +```markdown +# Dagster Ops Project + +This project demonstrates the same radiomics workflow using **Dagster Ops**. +Ops are imperative functions that define computation steps. They are connected in a job graph to form the pipeline. + +## Features +- Op-based orchestration for radiomics extraction. +- Explicit control of execution order via job definitions. +- Each op corresponds to one stage: image reading, registration, fusion, filtering, mask registration, feature extraction, and writing results. + +## Requirements +- Python 3.10+ +- Dagster +- SimpleITK +- Pandas +- PySERA + +Install dependencies: +```bash +pip install dagster simpleitk pandas pysera + +# Running: +dagster job execute -f dagster_op_runner.py +Or: +run.bat diff --git a/Dagster/Dagster_Op/app/__init__.py b/Dagster/Dagster_Op/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Dagster/Dagster_Op/app/__pycache__/__init__.cpython-311.pyc b/Dagster/Dagster_Op/app/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..635c05a Binary files /dev/null and b/Dagster/Dagster_Op/app/__pycache__/__init__.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/__pycache__/__init__.cpython-313.pyc b/Dagster/Dagster_Op/app/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..b174b68 Binary files /dev/null and b/Dagster/Dagster_Op/app/__pycache__/__init__.cpython-313.pyc differ diff --git a/Dagster/Dagster_Op/app/__pycache__/main.cpython-311.pyc b/Dagster/Dagster_Op/app/__pycache__/main.cpython-311.pyc new file mode 100644 index 0000000..0405f13 Binary files /dev/null and b/Dagster/Dagster_Op/app/__pycache__/main.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/__pycache__/main.cpython-313.pyc b/Dagster/Dagster_Op/app/__pycache__/main.cpython-313.pyc new file mode 100644 index 0000000..1caf980 Binary files /dev/null and b/Dagster/Dagster_Op/app/__pycache__/main.cpython-313.pyc differ diff --git a/Dagster/Dagster_Op/app/__pycache__/workflow_runner.cpython-311.pyc b/Dagster/Dagster_Op/app/__pycache__/workflow_runner.cpython-311.pyc new file mode 100644 index 0000000..a0913e7 Binary files /dev/null and b/Dagster/Dagster_Op/app/__pycache__/workflow_runner.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/engine/__pycache__/core.cpython-311.pyc b/Dagster/Dagster_Op/app/engine/__pycache__/core.cpython-311.pyc new file mode 100644 index 0000000..5ab7f4d Binary files /dev/null and b/Dagster/Dagster_Op/app/engine/__pycache__/core.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/engine/__pycache__/dagster_builder.cpython-311.pyc b/Dagster/Dagster_Op/app/engine/__pycache__/dagster_builder.cpython-311.pyc new file mode 100644 index 0000000..bbb5c62 Binary files /dev/null and b/Dagster/Dagster_Op/app/engine/__pycache__/dagster_builder.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/engine/__pycache__/dagster_runner.cpython-311.pyc b/Dagster/Dagster_Op/app/engine/__pycache__/dagster_runner.cpython-311.pyc new file mode 100644 index 0000000..16fcea3 Binary files /dev/null and b/Dagster/Dagster_Op/app/engine/__pycache__/dagster_runner.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/engine/__pycache__/radiuma_assets.cpython-311.pyc b/Dagster/Dagster_Op/app/engine/__pycache__/radiuma_assets.cpython-311.pyc new file mode 100644 index 0000000..3ce271f Binary files /dev/null and b/Dagster/Dagster_Op/app/engine/__pycache__/radiuma_assets.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/engine/__pycache__/radiuma_workflow_ops.cpython-311.pyc b/Dagster/Dagster_Op/app/engine/__pycache__/radiuma_workflow_ops.cpython-311.pyc new file mode 100644 index 0000000..48b2927 Binary files /dev/null and b/Dagster/Dagster_Op/app/engine/__pycache__/radiuma_workflow_ops.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/engine/core.py b/Dagster/Dagster_Op/app/engine/core.py new file mode 100644 index 0000000..5b0c353 --- /dev/null +++ b/Dagster/Dagster_Op/app/engine/core.py @@ -0,0 +1,72 @@ +import json +import threading +from pathlib import Path +from loguru import logger + +class WorkflowState: + def __init__(self, state_dir: Path): + self.state_dir = state_dir + self.state_file = state_dir / "state.json" + self._lock = threading.Lock() + self._state = {"steps": {}, "cancelled": False} + self.state_dir.mkdir(parents=True, exist_ok=True) + if self.state_file.exists(): + self._state = json.loads(self.state_file.read_text()) + + def mark_success(self, step_id: str, outputs: dict): + with self._lock: + self._state["steps"][step_id] = {"status": "success", "outputs": outputs} + self._write() + + def mark_failed(self, step_id: str, error: str): + with self._lock: + self._state["steps"][step_id] = {"status": "failed", "error": error} + self._write() + + def get_status(self, step_id: str): + return self._state["steps"].get(step_id, {}).get("status", "pending") + + def get_outputs(self, step_id: str): + return self._state["steps"].get(step_id, {}).get("outputs", {}) + + def set_cancelled(self, flag: bool): + with self._lock: + self._state["cancelled"] = flag + self._write() + + def is_cancelled(self) -> bool: + return self._state.get("cancelled", False) + + def _write(self): + self.state_file.write_text(json.dumps(self._state, indent=2)) + +class Node: + def __init__(self, step_id: str, run_fn, inputs: list, outputs: list): + self.step_id = step_id + self.run_fn = run_fn + self.inputs = inputs + self.outputs = outputs + +class Workflow: + def __init__(self, nodes: list[Node], state: WorkflowState): + self.nodes = nodes + self.state = state + + def execute(self): + for node in self.nodes: + status = self.state.get_status(node.step_id) + if status == "success": + logger.info(f"Skip {node.step_id}: already successful.") + continue + if self.state.is_cancelled(): + logger.warning("Execution cancelled. Stopping workflow.") + break + try: + logger.info(f"Run {node.step_id}") + outputs = node.run_fn(self.state) + self.state.mark_success(node.step_id, outputs) + logger.info(f"Success {node.step_id}") + except Exception as e: + logger.exception(f"Failed {node.step_id}: {e}") + self.state.mark_failed(node.step_id, str(e)) + break # Stop on failure, allow resume later diff --git a/Dagster/Dagster_Op/app/engine/dagster_builder.py b/Dagster/Dagster_Op/app/engine/dagster_builder.py new file mode 100644 index 0000000..2c46b1d --- /dev/null +++ b/Dagster/Dagster_Op/app/engine/dagster_builder.py @@ -0,0 +1,39 @@ +from dagster import op, job, In +import os, json +from app.features.pysera_node import run_pysera + +DATA_DIR = os.path.join("data", "images") +MASK_DIR = os.path.join("data", "masks") +OUTPUT_DIR = os.path.join("app", "storage", "artifacts") +os.makedirs(OUTPUT_DIR, exist_ok=True) + +@op(ins={"filename": In(str)}) +def load_image(filename: str) -> str: + # Return normalized image file path (do not load arrays/objects) + return os.path.normpath(os.path.join(DATA_DIR, filename)) + +@op(ins={"filename": In(str)}) +def load_mask(filename: str) -> str: + # Return normalized mask file path (do not load arrays/objects) + return os.path.normpath(os.path.join(MASK_DIR, filename)) + +@op +def extract_features(img_path: str, mask_path: str) -> str: + # Validate file paths and call PySERA + if not os.path.exists(img_path): + raise FileNotFoundError(f"Image path not found: {img_path}") + if not os.path.exists(mask_path): + raise FileNotFoundError(f"Mask path not found: {mask_path}") + + features = run_pysera(img_path, mask_path) + out_path = os.path.join(OUTPUT_DIR, "CT_pitch_features.json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(features, f, indent=2) + return out_path + +@job +def radiuma_job(): + # Compose ops: produce file paths, then extract features + img = load_image() + mask = load_mask() + extract_features(img, mask) diff --git a/Dagster/Dagster_Op/app/engine/dagster_runner.py b/Dagster/Dagster_Op/app/engine/dagster_runner.py new file mode 100644 index 0000000..dbbde97 --- /dev/null +++ b/Dagster/Dagster_Op/app/engine/dagster_runner.py @@ -0,0 +1,79 @@ +import os +import pysera +from app.engine import radiuma_assets +from app.engine.dagster_builder import radiuma_job +from dagster import materialize, FilesystemIOManager + +# Main workflow runner for Radiuma using PySERA and Dagster + +def run_workflow(): + # Define input image and mask paths + image_path = os.path.normpath("data/images/CT_pitch.nii.gz") + mask_path = os.path.normpath("data/masks/CT_pitch_mask.nii.gz") + output_dir = os.path.normpath("./results") + + # Run PySERA radiomics workflow + # Changed dimensions from "3d" to "1st,2D" + result = pysera.process_batch( + image_input=image_path, + mask_input=mask_path, + output_path=output_dir, + categories="all", + dimensions="1st,2D", # <-- only change applied here + apply_preprocessing=True, + ) + print("✅ PySERA workflow finished." if result.get("success") else "❌ PySERA workflow failed.") + + +def run_pipeline(): + # Run Dagster job with sample configuration + result = radiuma_job.execute_in_process( + run_config={ + "ops": { + "load_image": {"inputs": {"filename": {"value": "CT_pitch.nii.gz"}}}, + "load_mask": {"inputs": {"filename": {"value": "CT_pitch_mask.nii.gz"}}}, + } + } + ) + print("✅ Dagster job finished.") + print(result) + + +def run_assets(): + # Ensure artifacts directory exists + base_dir = os.path.normpath("app/storage/artifacts") + os.makedirs(base_dir, exist_ok=True) + + # Use FilesystemIOManager for asset storage + io_manager = FilesystemIOManager(base_dir=base_dir) + + # Materialize all assets defined in radiuma_assets + result = materialize( + [ + radiuma_assets.raw_image, + radiuma_assets.raw_mask, + radiuma_assets.registered_image, + radiuma_assets.fused_image, + radiuma_assets.features, + ], + resources={"io_manager": io_manager}, + ) + + if result.success: + print("✅ Assets materialized successfully.") + print(f"Storage base_dir: {base_dir}") + + # Print all materialized asset keys + for event in result.get_asset_materialization_events(): + print(f"Asset built: {event.asset_key}") + else: + print("❌ Asset execution failed.") + print(result) + + +if __name__ == "__main__": + print("🚀 Starting Radiuma unified run...\n") + run_workflow() + run_pipeline() + run_assets() + print("\n🎯 All workflows completed in one run.") diff --git a/Dagster/Dagster_Op/app/engine/radiuma_assets.py b/Dagster/Dagster_Op/app/engine/radiuma_assets.py new file mode 100644 index 0000000..a534420 --- /dev/null +++ b/Dagster/Dagster_Op/app/engine/radiuma_assets.py @@ -0,0 +1,57 @@ +import os +import SimpleITK as sitk +from dagster import asset + +DATA_DIR = os.path.join("data", "images") +MASK_DIR = os.path.join("data", "masks") +ARTIFACTS_DIR = os.path.join("app", "storage", "artifacts") +os.makedirs(ARTIFACTS_DIR, exist_ok=True) + +@asset +def raw_image() -> str: + # Return existing image path + path = os.path.normpath(os.path.join(DATA_DIR, "CT_pitch.nii.gz")) + if not os.path.exists(path): + raise FileNotFoundError(f"raw_image: file not found at {path}") + return path + +@asset +def raw_mask() -> str: + path = os.path.normpath(os.path.join(MASK_DIR, "CT_pitch_mask.nii.gz")) + if not os.path.exists(path): + raise FileNotFoundError(f"raw_mask: file not found at {path}") + return path + +@asset +def registered_image(raw_image: str) -> str: + # Read the raw image and write a registered copy + img = sitk.ReadImage(raw_image) + registered = sitk.Cast(img, img.GetPixelID()) # identity registration + out_path = os.path.normpath(os.path.join(ARTIFACTS_DIR, "CT_pitch_registered.nii.gz")) + sitk.WriteImage(registered, out_path) + return out_path + +@asset +def fused_image(registered_image: str, raw_mask: str) -> str: + img = sitk.ReadImage(registered_image) + mask = sitk.ReadImage(raw_mask) + mask_uint8 = sitk.Cast(mask, sitk.sitkUInt8) + fused = sitk.Mask(img, mask_uint8) + out_path = os.path.normpath(os.path.join(ARTIFACTS_DIR, "CT_pitch_fused.nii.gz")) + sitk.WriteImage(fused, out_path) + return out_path + +@asset +def features(fused_image: str) -> str: + import json, numpy as np + img = sitk.ReadImage(fused_image) + arr = sitk.GetArrayFromImage(img) + feats = { + "shape": list(arr.shape), + "spacing": list(img.GetSpacing()), + "intensity_sum": float(np.sum(arr)), + } + out_path = os.path.normpath(os.path.join(ARTIFACTS_DIR, "CT_pitch_asset_features.json")) + with open(out_path, "w", encoding="utf-8") as f: + json.dump(feats, f, indent=2) + return out_path \ No newline at end of file diff --git a/Dagster/Dagster_Op/app/engine/radiuma_workflow_ops.py b/Dagster/Dagster_Op/app/engine/radiuma_workflow_ops.py new file mode 100644 index 0000000..c7d9cad --- /dev/null +++ b/Dagster/Dagster_Op/app/engine/radiuma_workflow_ops.py @@ -0,0 +1,69 @@ +import json +import numpy as np +from dagster import op +import SimpleITK as sitk +from pathlib import Path +# Write Your codes Here: + + +ARTIFACTS = Path("artifacts") +ARTIFACTS.mkdir(exist_ok=True) + + +@op +def image_reader(path: str): + """Medical image reading (MRI, CT, PET).""" + img = sitk.ReadImage(path) + # Save the processed version for display in the GUI (optional) + sitk.WriteImage(img, str(ARTIFACTS / "processed_image.nii.gz")) + return img + +@op +def image_registration(fixed_img, moving_img): + """Aligning two medical images.""" + reg = sitk.ImageRegistrationMethod() + reg.SetMetricAsMeanSquares() + reg.SetOptimizerAsGradientDescent(learningRate=1.0, numberOfIterations=100) + reg.SetInterpolator(sitk.sitkLinear) + transform = reg.Execute(fixed_img, moving_img) + registered = sitk.Resample( + moving_img, fixed_img, transform, sitk.sitkLinear, 0.0, moving_img.GetPixelID() + ) + sitk.WriteImage(registered, str(ARTIFACTS / "registered_image.nii.gz")) + return registered + +@op +def image_fusion(img1, img2): + """Combining two medical images (simple fusion).""" + fused = sitk.Cast((img1 + img2) / 2, sitk.sitkFloat32) + sitk.WriteImage(fused, str(ARTIFACTS / "fused_image.nii.gz")) + return fused + +@op +def image_extraction(img): + """Extracting simple features from images.""" + arr = sitk.GetArrayFromImage(img) + features = { + "mean_intensity": float(np.mean(arr)), + "std_intensity": float(np.std(arr)), + "min_intensity": float(np.min(arr)), + "max_intensity": float(np.max(arr)), + } + (ARTIFACTS / "features.json").write_text(json.dumps(features, indent=2)) + return features + +@op +def image_writer(img): + """Save final output.""" + out_path = ARTIFACTS / "output.nii.gz" + sitk.WriteImage(img, str(out_path)) + return str(out_path) + +OPS = { + "reader": image_reader, + "registration": image_registration, + "fusion": image_fusion, + "extraction": image_extraction, + "writer": image_writer, +} + diff --git a/Dagster/Dagster_Op/app/features/__pycache__/pysera_node.cpython-311.pyc b/Dagster/Dagster_Op/app/features/__pycache__/pysera_node.cpython-311.pyc new file mode 100644 index 0000000..492a342 Binary files /dev/null and b/Dagster/Dagster_Op/app/features/__pycache__/pysera_node.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/features/pysera_node.py b/Dagster/Dagster_Op/app/features/pysera_node.py new file mode 100644 index 0000000..aef5623 --- /dev/null +++ b/Dagster/Dagster_Op/app/features/pysera_node.py @@ -0,0 +1,39 @@ +import os +import json +import numpy as np +import SimpleITK as sitk + +def run_pysera(image_path: str, mask_path: str): + # Read image and mask from file paths + image = sitk.ReadImage(str(image_path)) + mask = sitk.ReadImage(str(mask_path)) + + # Ensure mask is binary {0,1} and integer typed + mask = sitk.Cast(mask, sitk.sitkUInt8) + # Treat any non-zero voxel as 1 + mask = sitk.BinaryThreshold( + mask, + lowerThreshold=1, + upperThreshold=1_000_000, + insideValue=1, + outsideValue=0, + ) + + # Use proper radius vector for BinaryErode based on image dimension + dim = image.GetDimension() + radius = [1] * dim # e.g., [1,1,1] for 3D or [1,1] for 2D + + # Safe morphological operation on the binary mask + eroded = sitk.BinaryErode(mask, radius) + mask_array = sitk.GetArrayFromImage(mask) + eroded_array = sitk.GetArrayFromImage(eroded) + mask_border = np.logical_xor(mask_array, eroded_array) + + # Example features (replace with your real extraction pipeline) + features = { + "voxel_spacing": tuple(image.GetSpacing()), + "mask_voxels": int(np.sum(mask_array)), + "mask_border_voxels": int(np.sum(mask_border)), + } + + return features diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/connection_line.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/connection_line.cpython-311.pyc new file mode 100644 index 0000000..e99b42d Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/connection_line.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/graph_scene.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/graph_scene.cpython-311.pyc new file mode 100644 index 0000000..4f45f9f Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/graph_scene.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/gui_controls.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/gui_controls.cpython-311.pyc new file mode 100644 index 0000000..487f01e Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/gui_controls.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/napari_main.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/napari_main.cpython-311.pyc new file mode 100644 index 0000000..433b2c1 Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/napari_main.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/node_catalog.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/node_catalog.cpython-311.pyc new file mode 100644 index 0000000..9c68605 Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/node_catalog.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/node_widget.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/node_widget.cpython-311.pyc new file mode 100644 index 0000000..8f0fad8 Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/node_widget.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/pyside_features.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/pyside_features.cpython-311.pyc new file mode 100644 index 0000000..36839f9 Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/pyside_features.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/pyside_main.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/pyside_main.cpython-311.pyc new file mode 100644 index 0000000..159782a Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/pyside_main.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/visual_editor.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/visual_editor.cpython-311.pyc new file mode 100644 index 0000000..a3566a3 Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/visual_editor.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/__pycache__/visual_editor_step1.cpython-311.pyc b/Dagster/Dagster_Op/app/gui/__pycache__/visual_editor_step1.cpython-311.pyc new file mode 100644 index 0000000..51e01c9 Binary files /dev/null and b/Dagster/Dagster_Op/app/gui/__pycache__/visual_editor_step1.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/gui/connection_line.py b/Dagster/Dagster_Op/app/gui/connection_line.py new file mode 100644 index 0000000..fd0240b --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/connection_line.py @@ -0,0 +1,24 @@ +from PySide6.QtWidgets import QGraphicsLineItem, QMessageBox +from PySide6.QtGui import QPen, QColor + +class ConnectionLine(QGraphicsLineItem): + def __init__(self, source_port, target_port): + super().__init__() + self.source_port = source_port + self.target_port = target_port + self.update_position() + + # Validate connection types + if self.source_port.port_type != self.target_port.port_type: + # Invalid connection → red line + warning + self.setPen(QPen(QColor("red"), 2)) + QMessageBox.warning(None, "Invalid Connection", + f"Cannot connect {self.source_port.port_type} → {self.target_port.port_type}") + else: + # Valid connection → black line + self.setPen(QPen(QColor("blue"), 2)) + + def update_position(self): + src_pos = self.source_port.scenePos() + tgt_pos = self.target_port.scenePos() + self.setLine(src_pos.x(), src_pos.y(), tgt_pos.x(), tgt_pos.y()) diff --git a/Dagster/Dagster_Op/app/gui/graph_scene.py b/Dagster/Dagster_Op/app/gui/graph_scene.py new file mode 100644 index 0000000..5070314 --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/graph_scene.py @@ -0,0 +1,214 @@ +from PySide6.QtWidgets import QGraphicsScene, QGraphicsEllipseItem, QGraphicsLineItem +from PySide6.QtGui import QBrush, QColor, QPen +from PySide6.QtCore import QPointF +from app.gui.node_widget import NodeWidget + + +class Port(QGraphicsEllipseItem): + """An input/output port that is placed next to each node.""" + def __init__(self, parent_node: NodeWidget, kind: str, offset_x: float, offset_y: float): + super().__init__(-5, -5, 10, 10, parent_node) + self.parent_node = parent_node + self.kind = kind # "input" Or "output" + self.setBrush(QBrush(QColor("#2e7d32") if kind == "input" else QColor("#1565c0"))) + self.setPos(parent_node.rect().x() + offset_x, parent_node.rect().y() + offset_y) + self.setZValue(1.0) + + +class ConnectionLine(QGraphicsLineItem): + """Connection between two ports.""" + def __init__(self, source_port: Port, target_port: Port): + super().__init__() + self.source_port = source_port + self.target_port = target_port + pen = QPen(QColor("#555")) + pen.setWidth(2) + self.setPen(pen) + self.update_positions() + + def update_positions(self): + src = self.source_port.mapToScene(QPointF(0, 0)) + tgt = self.target_port.mapToScene(QPointF(0, 0)) + self.setLine(src.x(), src.y(), tgt.x(), tgt.y()) + + +class GraphScene(QGraphicsScene): + """Graph scene with nodes, ports, and connections. Only connected nodes are exported.""" + def __init__(self): + super().__init__() + self.connections: list[ConnectionLine] = [] + # Manual connection mode: You click on an outgoing port, then click on the destination incoming port. + self._pending_source_port: Port | None = None + + # ---------- Helper tool for adding default nodes ---------- + def add_default_medical_nodes(self): + """Adds five medical nodes and installs their ports.""" + # Create nodes + reader = NodeWidget("Image Reader", [], ["output"]) + registration = NodeWidget("Image Registration", ["input"], ["output"]) + fusion = NodeWidget("Image Fusion", ["input1", "input2"], ["output"]) + extraction = NodeWidget("Image Extraction", ["input"], ["output"]) + writer = NodeWidget("Image Writer", ["input"], []) + + # Initial placement + reader.setPos(50, 50) + registration.setPos(250, 50) + fusion.setPos(450, 50) + extraction.setPos(650, 50) + writer.setPos(850, 50) + + # Add to scene + for n in (reader, registration, fusion, extraction, writer): + self.addItem(n) + + # Install ports for each node (a simple I/O is enough) + # Reader: Output only + reader.output_port = Port(reader, "output", 150, 45) + + # Registration: Input + Output + registration.input_port = Port(registration, "input", 0, 45) + registration.output_port = Port(registration, "output", 150, 45) + + # Fusion: Two inputs + one output + fusion.input_port_1 = Port(fusion, "input", 0, 30) + fusion.input_port_2 = Port(fusion, "input", 0, 60) + fusion.output_port = Port(fusion, "output", 150, 45) + + # Extraction: Input + Output + extraction.input_port = Port(extraction, "input", 0, 45) + extraction.output_port = Port(extraction, "output", 150, 45) + + # Writer: Input only + writer.input_port = Port(writer, "input", 0, 45) + + # Register the port click handle for manual connection + self._install_port_handlers([reader, registration, fusion, extraction, writer]) + + def _install_port_handlers(self, nodes: list[NodeWidget]): + """Mouse handles for ports so the user can make connections.""" + all_ports: list[Port] = [] + for n in nodes: + for attr in dir(n): + if attr.endswith("port"): + p = getattr(n, attr) + if isinstance(p, Port): + all_ports.append(p) + + for port in all_ports: + port.mousePressEvent = lambda event, p=port: self._on_port_clicked(p) + + def _on_port_clicked(self, port: Port): + """Ports click logic: output first, then input; connection is made.""" + if port.kind == "output": + # Start connection + self._pending_source_port = port + elif port.kind == "input" and self._pending_source_port is not None: + # Connection completion + line = ConnectionLine(self._pending_source_port, port) + self.addItem(line) + self.connections.append(line) + # Update positions when nodes move + self._pending_source_port = None + + # ----------Export/Load Graph ---------- + def export_graph(self): + """Exports only nodes involved in connections.""" + data = {"nodes": [], "connections": []} + used_nodes = set() + + for conn in self.connections: + src_node = conn.source_port.parent_node + tgt_node = conn.target_port.parent_node + used_nodes.add(src_node) + used_nodes.add(tgt_node) + data["connections"].append({ + "source": src_node.title.toPlainText(), + "target": tgt_node.title.toPlainText() + }) + + for node in used_nodes: + data["nodes"].append({ + "id": node.title.toPlainText(), # Persistent ID based on title + "type": node.title.toPlainText(), + "pos": [node.pos().x(), node.pos().y()] + }) + + return data + + def save_graph(self, filename: str): + import json + from pathlib import Path + Path(filename).write_text(json.dumps(self.export_graph(), indent=2)) + + def load_graph(self, filename: str): + """Rebuild ports and connections from saved file. + Note: Here we assume that the default nodes are present on the scene; we only draw the connections. + """ + import json + from pathlib import Path + if not Path(filename).exists(): + return + data = json.loads(Path(filename).read_text()) + + # Create a mapping from title → node in the scene + title_to_node = {} + for item in self.items(): + if isinstance(item, NodeWidget): + title_to_node[item.title.toPlainText()] = item + + # Rebuild connections + self._rebuild_ports_if_missing(title_to_node) + for conn in data.get("connections", []): + src = title_to_node.get(conn["source"]) + tgt = title_to_node.get(conn["target"]) + if not src or not tgt: + continue + + # Select default input/output port based on node name + src_port = getattr(src, "output_port", None) + tgt_port = getattr(tgt, "input_port", None) + + # Fusion has two inputs; if the destination is Fusion and the first input is busy, take the second input + if tgt.title.toPlainText() == "Image Fusion": + # If there is no previous connection to input_port_1, connect to it; otherwise connect to input_port_2 + candidates = [tgt.input_port_1, tgt.input_port_2] + tgt_port = candidates[0] + for c in self.connections: + if c.target_port is candidates[0]: + tgt_port = candidates[1] + break + + if src_port and tgt_port: + line = ConnectionLine(src_port, tgt_port) + self.addItem(line) + self.connections.append(line) + + def _rebuild_ports_if_missing(self, title_to_node: dict): + """If the node ports haven't been created yet for some reason, we'll create them here.""" + for title, n in title_to_node.items(): + # Reader + if title == "Image Reader" and not hasattr(n, "output_port"): + n.output_port = Port(n, "output", 150, 45) + # Registration + if title == "Image Registration": + if not hasattr(n, "input_port"): + n.input_port = Port(n, "input", 0, 45) + if not hasattr(n, "output_port"): + n.output_port = Port(n, "output", 150, 45) + # Fusion + if title == "Image Fusion": + if not hasattr(n, "input_port_1"): + n.input_port_1 = Port(n, "input", 0, 30) + if not hasattr(n, "input_port_2"): + n.input_port_2 = Port(n, "input", 0, 60) + if not hasattr(n, "output_port"): + n.output_port = Port(n, "output", 150, 45) + # Extraction + if title == "Image Extraction": + if not hasattr(n, "input_port"): + n.input_port = Port(n, "input", 0, 45) + if not hasattr(n, "output_port"): + n.output_port = Port(n, "output", 150, 45) + # Writer + if title == "Image Writer" and not hasattr(n, "input_port"): + n.input_port = Port(n, "input", 0, 45) \ No newline at end of file diff --git a/Dagster/Dagster_Op/app/gui/gui_controls.py b/Dagster/Dagster_Op/app/gui/gui_controls.py new file mode 100644 index 0000000..f47129e --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/gui_controls.py @@ -0,0 +1,60 @@ +from PySide6.QtWidgets import QPushButton, QHBoxLayout, QWidget +from dagster_radiuma.jobs import radiomics_job, radiomics_batch_job + +class WorkflowControls(QWidget): + def __init__(self, log_panel, history_panel, parent=None): + super().__init__(parent) + + self.log_panel = log_panel + self.history_panel = history_panel + + # دکمه‌ها + self.start_btn = QPushButton("Start") + self.stop_btn = QPushButton("Stop") + self.resume_btn = QPushButton("Resume") + self.cancel_btn = QPushButton("Cancel") + + layout = QHBoxLayout() + layout.addWidget(self.start_btn) + layout.addWidget(self.stop_btn) + layout.addWidget(self.resume_btn) + layout.addWidget(self.cancel_btn) + self.setLayout(layout) + + # اتصال + self.start_btn.clicked.connect(self.start_workflow) + self.stop_btn.clicked.connect(self.stop_workflow) + self.resume_btn.clicked.connect(self.resume_workflow) + self.cancel_btn.clicked.connect(self.cancel_workflow) + + self.current_run_id = None + + def log(self, message): + if hasattr(self.log_panel, "append"): + self.log_panel.append(message) + if hasattr(self.history_panel, "append"): + self.history_panel.append(message) + + def start_workflow(self): + self.log("[GUI] Start → Dagster job starting...") + result = radiomics_job.execute_in_process() + self.current_run_id = result.run_id + self.log(f"[GUI] Workflow started. Success={result.success}") + + def stop_workflow(self): + if self.current_run_id: + self.log(f"[GUI] Stop → Terminating run {self.current_run_id}") + # dagster_api.terminate_run(self.current_run_id) + self.log("[GUI] Workflow stopped.") + + def resume_workflow(self): + if self.current_run_id: + self.log(f"[GUI] Resume → Restarting run {self.current_run_id}") + # dagster_api.resume_run(self.current_run_id) + self.log("[GUI] Workflow resumed. [Recovered]") + + def cancel_workflow(self): + if self.current_run_id: + self.log(f"[GUI] Cancel → Cancelling run {self.current_run_id}") + # dagster_api.cancel_run(self.current_run_id) + self.log("[GUI] Workflow cancelled.") diff --git a/Dagster/Dagster_Op/app/gui/napari_main.py b/Dagster/Dagster_Op/app/gui/napari_main.py new file mode 100644 index 0000000..276f120 --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/napari_main.py @@ -0,0 +1,24 @@ +# app/gui/napari_main.py +import napari +from pathlib import Path +import SimpleITK as sitk +import numpy as np + +def sitk_to_numpy(img): + return sitk.GetArrayFromImage(img) # z, y, x + +def run_napari_viewer(): + viewer = napari.Viewer(title="Radiuma Mini - Viewer") + # Load outputs if they exist + filtered = Path("app/storage/artifacts/filtered.nii.gz") + mask = Path("data/masks/CT_AVM_mask.nii.gz") + if filtered.exists(): + arr = sitk_to_numpy(sitk.ReadImage(str(filtered))) + viewer.add_image(arr, name="filtered", blending="translucent") + if mask.exists(): + marr = sitk_to_numpy(sitk.ReadImage(str(mask))) + viewer.add_labels(marr, name="mask") + napari.run() + +if __name__ == "__main__": + run_napari_viewer() diff --git a/Dagster/Dagster_Op/app/gui/node_catalog.py b/Dagster/Dagster_Op/app/gui/node_catalog.py new file mode 100644 index 0000000..840d51d --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/node_catalog.py @@ -0,0 +1,368 @@ +import SimpleITK as sitk +import numpy as np +import pandas as pd +import pysera +from NodeGraphQt import BaseNode +from PySide6.QtWidgets import QWidget, QVBoxLayout, QLineEdit, QTreeWidget, QTreeWidgetItem +from PySide6.QtCore import Qt +from PySide6.QtGui import QIcon + + +# ------------------------------- +# Helpers for safe input resolution +# ------------------------------- + +def _resolve_array_from_input(node: BaseNode, port_index: int, upstream_attr_name: str): + """ + Try to get a NumPy array from node.get_input(port_index). + If it's a Port (NodeGraphQt Port), traverse to the connected upstream node + and read the given upstream_attr_name as a fallback payload. + """ + val = node.get_input(port_index) + # direct array + if isinstance(val, np.ndarray): + return val + # try to traverse port connections + try: + if hasattr(val, 'connected_ports'): + ports = val.connected_ports() + if ports: + upstream_node = ports[0].node() + upstream_val = getattr(upstream_node, upstream_attr_name, None) + if isinstance(upstream_val, np.ndarray): + return upstream_val + except Exception: + pass + return None + + +def _resolve_features_from_input(node: BaseNode, port_index: int, upstream_attr_name: str): + """ + Resolve features payload from input. Accepts pandas.DataFrame, dict, list, or str. + If input returns a Port, read upstream node attribute. + """ + val = node.get_input(port_index) + # direct usable types + if hasattr(val, "to_csv"): # pandas DataFrame + return val + if isinstance(val, (dict, list, str)): + return val + # fallback via upstream + try: + if hasattr(val, 'connected_ports'): + ports = val.connected_ports() + if ports: + upstream_node = ports[0].node() + upstream_val = getattr(upstream_node, upstream_attr_name, None) + if hasattr(upstream_val, "to_csv") or isinstance(upstream_val, (dict, list, str)): + return upstream_val + except Exception: + pass + return None + + +# ------------------------------- +# Node Definitions +# ------------------------------- + +class ImageReaderNode(BaseNode): + __identifier__ = 'image.io' + NODE_NAME = 'Image Reader' + + def __init__(self): + super().__init__() + self.add_output('image') + self.add_output('mask') + # store payloads for upstream fallback resolution + self._image_array = None + self._mask_array = None + + def run(self): + try: + image_path = r"C:/Users/Omen16/Documents/Radiuma_Mini/data/images/CT_AVM.nii.gz" + mask_path = r"C:/Users/Omen16/Documents/Radiuma_Mini/data/masks/CT_AVM_mask.nii.gz" + + image = sitk.ReadImage(image_path) + mask = sitk.ReadImage(mask_path) + + img_arr = sitk.GetArrayFromImage(image) + msk_arr = sitk.GetArrayFromImage(mask) + + # save locally and on ports + self._image_array = img_arr + self._mask_array = msk_arr + + print("Reader: loaded image & mask.") + self.set_output(0, img_arr) + self.set_output(1, msk_arr) + except Exception as e: + print("Reader error:", e) + + +class ImageWriterNode(BaseNode): + __identifier__ = 'image.io' + NODE_NAME = 'Image Writer' + + def __init__(self): + super().__init__() + self.add_input('image') + self.add_input('features') + + def run(self): + # resolve image array robustly (if input is a Port, traverse upstream) + image_array = _resolve_array_from_input(self, 0, '_image_array') + features = _resolve_features_from_input(self, 1, '_features') + + # image save + if isinstance(image_array, np.ndarray) and image_array.size > 0: + try: + sitk_image = sitk.GetImageFromArray(image_array) + sitk.WriteImage(sitk_image, "output_image.nii.gz") + print("Writer: image saved.") + except Exception as e: + print("Writer image save error:", e) + else: + print("Writer: no image to save.") + + # features save + if features is not None: + try: + if hasattr(features, "to_csv"): + features.to_csv("features.csv", index=False) + elif isinstance(features, dict): + pd.DataFrame([features]).to_csv("features.csv", index=False) + elif isinstance(features, list): + pd.DataFrame(features).to_csv("features.csv", index=False) + else: + pd.DataFrame([{"features": str(features)}]).to_csv("features.csv", index=False) + print("Writer: features saved.") + except Exception as e: + print("Writer features save error:", e) + else: + print("Writer: no features to save.") + + +class ImageFilterNode(BaseNode): + __identifier__ = 'image.proc' + NODE_NAME = 'Image Filter' + + def __init__(self): + super().__init__() + self.add_input('image') + self.add_output('filtered_image') + self._filtered_array = None + + def run(self): + # resolve input image array robustly + image_array = _resolve_array_from_input(self, 0, '_image_array') + if not isinstance(image_array, np.ndarray) or image_array.size == 0: + print("Filter: no valid image.") + return + try: + image = sitk.GetImageFromArray(image_array) + filtered = sitk.SmoothingRecursiveGaussian(image, sigma=2.0) + filtered_arr = sitk.GetArrayFromImage(filtered) + + self._filtered_array = filtered_arr + self.set_output(0, filtered_arr) + print("Filter: applied Gaussian smoothing.") + except Exception as e: + print("Filter error:", e) + + +class ImageRegistrationNode(BaseNode): + __identifier__ = 'image.proc' + NODE_NAME = 'Image Registration' + + def __init__(self): + super().__init__() + self.add_input('fixed_image') + self.add_input('moving_image') + self.add_output('registered_image') + self._registered_array = None + + def run(self): + fixed_array = _resolve_array_from_input(self, 0, '_image_array') + moving_array = _resolve_array_from_input(self, 1, '_image_array') + + if not isinstance(fixed_array, np.ndarray) or not isinstance(moving_array, np.ndarray): + print("Registration: invalid inputs.") + return + + try: + fixed = sitk.GetImageFromArray(fixed_array) + moving = sitk.GetImageFromArray(moving_array) + + registration = sitk.ImageRegistrationMethod() + registration.SetMetricAsMeanSquares() + registration.SetOptimizerAsGradientDescent( + learningRate=1.0, + numberOfIterations=50 + ) + registration.SetInterpolator(sitk.sitkLinear) + + initial_transform = sitk.CenteredTransformInitializer( + fixed, + moving, + sitk.Euler3DTransform(), + sitk.CenteredTransformInitializerFilter.GEOMETRY + ) + registration.SetInitialTransform(initial_transform, inPlace=False) + + final_transform = registration.Execute(fixed, moving) + registered = sitk.Resample( + moving, fixed, final_transform, + sitk.sitkLinear, 0.0, moving.GetPixelID() + ) + + reg_arr = sitk.GetArrayFromImage(registered) + self._registered_array = reg_arr + self.set_output(0, reg_arr) + print("Registration: complete.") + except Exception as e: + print("Registration error:", e) + + +class ImageFusionNode(BaseNode): + __identifier__ = 'image.proc' + NODE_NAME = 'Image Fusion' + + def __init__(self): + super().__init__() + self.add_input('image_A') + self.add_input('image_B') + self.add_output('fused_image') + self._fused_array = None + + def run(self): + img_a = _resolve_array_from_input(self, 0, '_image_array') + img_b = _resolve_array_from_input(self, 1, '_image_array') + + if not isinstance(img_a, np.ndarray) or not isinstance(img_b, np.ndarray): + print("Fusion: invalid inputs.") + return + + try: + fused = (img_a.astype(np.float32) * 0.5 + img_b.astype(np.float32) * 0.5) + self._fused_array = fused + self.set_output(0, fused) + print("Fusion: complete.") + except Exception as e: + print("Fusion error:", e) + + +class FeatureExtractionNode(BaseNode): + __identifier__ = 'analysis' + NODE_NAME = 'Feature Extraction' + + def __init__(self): + super().__init__() + self.add_input('image') + self.add_input('mask') + self.add_output('features') + self._features = None + + def run(self): + image_array = _resolve_array_from_input(self, 0, '_image_array') + mask_array = _resolve_array_from_input(self, 1, '_mask_array') + + if not isinstance(image_array, np.ndarray) or not isinstance(mask_array, np.ndarray): + print("Extraction: invalid image/mask.") + return + + try: + result = pysera.process_batch( + image_input=image_array, + mask_input=mask_array, + output_path="./results", + categories="all", + dimensions="3d", + apply_preprocessing=True + ) + features = result.get("features_extracted") + self._features = features + self.set_output(0, features) + + # End of processing report + print(f"Extraction finished. Success={result.get('success')}, " + f"Features={len(features) if features is not None else 0}, " + f"Time={result.get('processing_time')}s") + + except Exception as e: + print("PySERA error:", e) + + +# ------------------------------- +# Node Catalog Widget +# ------------------------------- + +class NodeCatalog(QWidget): + def __init__(self, graph): + super().__init__() + self.graph = graph + self.node_count = 0 + + layout = QVBoxLayout(self) + + self.search_bar = QLineEdit() + self.search_bar.setPlaceholderText("Search nodes...") + layout.addWidget(self.search_bar) + + self.tree = QTreeWidget() + self.tree.setHeaderHidden(True) + layout.addWidget(self.tree) + + self.categories = { + "Image I/O": [ + (ImageReaderNode, "icons/camera.png"), + (ImageWriterNode, "icons/save.png") + ], + "Image Processing": [ + (ImageFilterNode, "icons/filter.png"), + (ImageRegistrationNode, "icons/transform.png"), + (ImageFusionNode, "icons/fusion.png") + ], + "Feature Extraction": [ + (FeatureExtractionNode, "icons/chart.png") + ] + } + + self.populate_tree() + self.search_bar.textChanged.connect(self.filter_nodes) + self.tree.itemDoubleClicked.connect(self.add_node) + + def populate_tree(self): + self.tree.clear() + for category, nodes in self.categories.items(): + cat_item = QTreeWidgetItem([category]) + self.tree.addTopLevelItem(cat_item) + for node_cls, icon_path in nodes: + node_item = QTreeWidgetItem([node_cls.NODE_NAME]) + node_item.setData(0, Qt.UserRole, node_cls) + node_item.setIcon(0, QIcon(icon_path)) + cat_item.addChild(node_item) + + def filter_nodes(self, text): + text = text.lower() + for i in range(self.tree.topLevelItemCount()): + cat_item = self.tree.topLevelItem(i) + visible_cat = False + for j in range(cat_item.childCount()): + node_item = cat_item.child(j) + node_name = node_item.text(0).lower() + is_match = text in node_name + node_item.setHidden(not is_match) + if is_match: + visible_cat = True + cat_item.setHidden(not visible_cat) + + def add_node(self, item, column): + node_cls = item.data(0, Qt.UserRole) + if node_cls: + node = node_cls() + # grid placement to avoid overlap + spacing_x, spacing_y = 200, 120 + x = spacing_x * (self.node_count % 4) + y = spacing_y * (self.node_count // 4) + self.graph.add_node(node, pos=(x, y)) + self.node_count += 1 diff --git a/Dagster/Dagster_Op/app/gui/node_widget.py b/Dagster/Dagster_Op/app/gui/node_widget.py new file mode 100644 index 0000000..7017fd3 --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/node_widget.py @@ -0,0 +1,133 @@ +from PySide6.QtWidgets import ( + QMessageBox, QGraphicsRectItem, QGraphicsTextItem, + QPushButton, QGraphicsProxyWidget, QTableWidget, + QTableWidgetItem, QDialog, QVBoxLayout +) +from PySide6.QtGui import QBrush, QColor +from pathlib import Path +import napari +import json +from .port_item import PortItem # Import PortItem with type support + +class NodeWidget(QGraphicsRectItem): + def __init__(self, title: str, inputs: list[tuple[str, str]], outputs: list[tuple[str, str]]): + """ + inputs/outputs: list of (label, port_type) + Example: + inputs=[("Image", "Image"), ("Mask", "Mask")] + outputs=[("Features", "Features")] + """ + super().__init__(0, 0, 160, 90) + self.setBrush(QBrush(QColor("#e0e0e0"))) + self.title = QGraphicsTextItem(title, self) + self.title.setPos(10, 5) + + self.status_text = QGraphicsTextItem("", self) + self.status_text.setPos(10, 40) + + # Create input ports + self.input_ports = [] + y_offset = 20 + for label, port_type in inputs: + port = PortItem(label, port_type, is_input=True, parent=self) + port.setPos(0, y_offset) + self.input_ports.append(port) + y_offset += 20 + + # Create output ports + self.output_ports = [] + y_offset = 20 + for label, port_type in outputs: + port = PortItem(label, port_type, is_input=False, parent=self) + port.setPos(150, y_offset) + self.output_ports.append(port) + y_offset += 20 + + # Output Display Button + self.btn_view = QPushButton("View Output") + proxy = QGraphicsProxyWidget(self) + proxy.setWidget(self.btn_view) + proxy.setPos(10, 60) + self.btn_view.clicked.connect(self.view_output) + + def set_status(self, status: str): + # Change node color and status text based on execution state + if status == "running": + self.setBrush(QBrush(QColor("yellow"))) + self.status_text.setPlainText("⏳ Running") + elif status == "success": + self.setBrush(QBrush(QColor("lightgreen"))) + self.status_text.setPlainText("✔ Success") + elif status == "failed": + self.setBrush(QBrush(QColor("red"))) + self.status_text.setPlainText("❌ Failed") + else: + self.setBrush(QBrush(QColor("#e0e0e0"))) + self.status_text.setPlainText("") + + def view_output(self): + node_type = self.title.toPlainText() + + # Image Reader Or Image Filter → Processed Image + if node_type in ["Image Reader", "Image Filter"]: + file_path = Path("artifacts/processed_image.nii.gz") + if file_path.exists(): + viewer = napari.Viewer() + viewer.open(str(file_path)) + else: + QMessageBox.warning(None, "Output not found", f"No output image for {node_type}") + + # Mask Reader → Mask + elif node_type == "Mask Reader": + file_path = Path("artifacts/mask.nii.gz") + if file_path.exists(): + viewer = napari.Viewer() + viewer.open(str(file_path)) + else: + QMessageBox.warning(None, "Output not found", "No mask file found") + + # Image Registration → Registered image + elif node_type == "Image Registration": + file_path = Path("artifacts/registered_image.nii.gz") + if file_path.exists(): + viewer = napari.Viewer() + viewer.open(str(file_path)) + else: + QMessageBox.warning(None, "Output not found", "No registered image found") + + # Image Fusion → Composite Image + elif node_type == "Image Fusion": + file_path = Path("artifacts/fused_image.nii.gz") + if file_path.exists(): + viewer = napari.Viewer() + viewer.open(str(file_path)) + else: + QMessageBox.warning(None, "Output not found", "No fused image found") + + # Image Extraction Or PySERA Extract → Features table + elif node_type in ["Image Extraction", "PySERA Extract"]: + file_path = Path("artifacts/features.json") + if file_path.exists(): + features = json.loads(file_path.read_text()) + dialog = QDialog() + dialog.setWindowTitle("Radiomics Features") + layout = QVBoxLayout(dialog) + table = QTableWidget() + table.setRowCount(len(features)) + table.setColumnCount(2) + table.setHorizontalHeaderLabels(["Feature", "Value"]) + for i, (key, value) in enumerate(features.items()): + table.setItem(i, 0, QTableWidgetItem(key)) + table.setItem(i, 1, QTableWidgetItem(str(value))) + layout.addWidget(table) + dialog.exec() + else: + QMessageBox.warning(None, "Output not found", "No features.json found") + + # Writer → Final Output File + elif node_type in ["Writer", "Image Writer"]: + file_path = Path("artifacts/output.nii.gz") + if file_path.exists(): + QMessageBox.information(None, "Writer Output", f"Output file saved: {file_path}") + else: + QMessageBox.warning(None, "Output not found", "No output file found") diff --git a/Dagster/Dagster_Op/app/gui/port_item.py b/Dagster/Dagster_Op/app/gui/port_item.py new file mode 100644 index 0000000..a42ce13 --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/port_item.py @@ -0,0 +1,14 @@ +from PySide6.QtWidgets import QGraphicsEllipseItem, QGraphicsTextItem +from PySide6.QtGui import QBrush, QColor + +class PortItem(QGraphicsEllipseItem): + def __init__(self, label: str, port_type: str, is_input: bool, parent=None): + super().__init__(-5, -5, 10, 10, parent) + # Color based on input/output + self.setBrush(QBrush(QColor("green") if is_input else QColor("blue"))) + self.label = QGraphicsTextItem(label, parent) + self.label.setDefaultTextColor(QColor("black")) + self.label.setPos(10 if is_input else -50, -7) + self.is_input = is_input + # Port type (e.g. "Image", "Mask", "Features") + self.port_type = port_type diff --git a/Dagster/Dagster_Op/app/gui/pyside_features.py b/Dagster/Dagster_Op/app/gui/pyside_features.py new file mode 100644 index 0000000..bc4b3c7 --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/pyside_features.py @@ -0,0 +1,53 @@ +import sys +import json +from pathlib import Path +from PySide6.QtWidgets import QApplication, QWidget, QVBoxLayout, QPushButton, QLabel, QTableWidget, QTableWidgetItem + +class FeatureViewer(QWidget): + def __init__(self): + super().__init__() + self.setWindowTitle("Radiomics Features Viewer") + layout = QVBoxLayout(self) + + self.status = QLabel("Load features.json to view results") + self.table = QTableWidget() + self.btn_load = QPushButton("Load Features") + + layout.addWidget(self.status) + layout.addWidget(self.table) + layout.addWidget(self.btn_load) + + self.btn_load.clicked.connect(self.load_features) + + def load_features(self): + features_path = Path("artifacts/features.json") + if not features_path.exists(): + self.status.setText("No features.json found! Run workflow first.") + return + + try: + features = json.loads(features_path.read_text()) + except Exception as e: + self.status.setText(f"Error reading features.json: {e}") + return + + # Displaying features in a table + self.table.setRowCount(len(features)) + self.table.setColumnCount(2) + self.table.setHorizontalHeaderLabels(["Feature", "Value"]) + + for i, (key, value) in enumerate(features.items()): + self.table.setItem(i, 0, QTableWidgetItem(key)) + self.table.setItem(i, 1, QTableWidgetItem(str(value))) + + self.status.setText("Features loaded successfully!") + +def run_gui(): + app = QApplication(sys.argv) + viewer = FeatureViewer() + viewer.resize(500, 400) + viewer.show() + sys.exit(app.exec()) + +if __name__ == "__main__": + run_gui() diff --git a/Dagster/Dagster_Op/app/gui/pyside_main.py b/Dagster/Dagster_Op/app/gui/pyside_main.py new file mode 100644 index 0000000..b097783 --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/pyside_main.py @@ -0,0 +1,61 @@ +import sys +from pathlib import Path +from PySide6.QtWidgets import QApplication, QWidget, QVBoxLayout, QPushButton, QLabel, QFileDialog +from app.engine.dagster_runner import run_workflow_with_config +# Write Your Code Here: + +class MainWindow(QWidget): + def __init__(self): + super().__init__() + self.setWindowTitle("Radiuma Mini - Workflow Runner") + layout = QVBoxLayout(self) + + self.status = QLabel("Idle") + self.btn_select_image = QPushButton("Select Image File") + self.btn_select_mask = QPushButton("Select Mask File") + self.btn_run = QPushButton("Run Workflow") + + layout.addWidget(self.status) + layout.addWidget(self.btn_select_image) + layout.addWidget(self.btn_select_mask) + layout.addWidget(self.btn_run) + + self.image_file = None + self.mask_file = None + + self.btn_select_image.clicked.connect(self.select_image) + self.btn_select_mask.clicked.connect(self.select_mask) + self.btn_run.clicked.connect(self.run_workflow) + + def select_image(self): + file_path, _ = QFileDialog.getOpenFileName(self, "Select Image", "data/images", "NIfTI files (*.nii.gz)") + if file_path: + self.image_file = Path(file_path).name + self.status.setText(f"Selected image: {self.image_file}") + + def select_mask(self): + file_path, _ = QFileDialog.getOpenFileName(self, "Select Mask", "data/masks", "NIfTI files (*.nii.gz)") + if file_path: + self.mask_file = Path(file_path).name + self.status.setText(f"Selected mask: {self.mask_file}") + + def run_workflow(self): + if not self.image_file or not self.mask_file: + self.status.setText("Please select both image and mask files!") + return + self.status.setText("Running workflow...") + result = run_workflow_with_config(self.image_file, self.mask_file) + if result.success: + self.status.setText("Workflow completed successfully!") + else: + self.status.setText("Workflow failed!") + +def run_gui(): + app = QApplication(sys.argv) + w = MainWindow() + w.resize(400, 200) + w.show() + sys.exit(app.exec()) + +if __name__ == "__main__": + run_gui() diff --git a/Dagster/Dagster_Op/app/gui/visual_editor.py b/Dagster/Dagster_Op/app/gui/visual_editor.py new file mode 100644 index 0000000..725bf7a --- /dev/null +++ b/Dagster/Dagster_Op/app/gui/visual_editor.py @@ -0,0 +1,179 @@ +import sys +import datetime +from PySide6.QtWidgets import ( + QApplication, QMainWindow, QSplitter, QTextEdit, QPushButton, QWidget, QVBoxLayout +) +from PySide6.QtCore import Qt +from PySide6.QtGui import QColor, QTextCharFormat, QTextCursor + +from NodeGraphQt import NodeGraph +from NodeGraphQt.errors import NodeRegistrationError + +from dagster import DagsterInstance, execute_job +import yaml +from dagster_radiuma.jobs import radiomics_job, radiomics_batch_job + +from app.gui.node_catalog import ( + NodeCatalog, + ImageReaderNode, + ImageWriterNode, + ImageRegistrationNode, + ImageFilterNode, + ImageFusionNode, + FeatureExtractionNode +) + +from app.gui.gui_controls import WorkflowControls + + +class VisualEditor(QMainWindow): + def __init__(self): + super().__init__() + self.setWindowTitle("Radiuma Workflow Editor") + self.resize(1200, 800) + + # Graph + self.graph = NodeGraph() + + # Register nodes with unique alias: identifier + ClassName + def reg(cls): + key = f"{cls.__identifier__}.{cls.__name__}" + try: + self.graph.register_node(cls, alias=key) + except NodeRegistrationError: + pass + + reg(ImageReaderNode) + reg(ImageWriterNode) + reg(ImageRegistrationNode) + reg(ImageFilterNode) + reg(ImageFusionNode) + reg(FeatureExtractionNode) + + # Viewer + self.viewer = self.graph.widget + + # Node catalog + self.catalog = NodeCatalog(self.graph) + + # Log panel + self.log_panel = QTextEdit() + self.log_panel.setReadOnly(True) + self.log_panel.setPlaceholderText("Logs will appear here...") + + # Workflow controls (Start/Stop/Resume/Cancel) + self.controls = WorkflowControls(self.log_panel, self.log_panel) + # اتصال متدهای کنترل به Dagster + self.controls.start_btn.clicked.connect(self.start_workflow) + self.controls.stop_btn.clicked.connect(self.stop_workflow) + self.controls.resume_btn.clicked.connect(self.resume_workflow) + self.controls.cancel_btn.clicked.connect(self.cancel_workflow) + + # Left panel layout + left_panel = QWidget() + left_layout = QVBoxLayout(left_panel) + left_layout.addWidget(self.catalog) + left_layout.addWidget(self.controls) + + # Splitters + top_splitter = QSplitter(Qt.Horizontal) + top_splitter.addWidget(left_panel) + top_splitter.addWidget(self.viewer) + + main_splitter = QSplitter(Qt.Vertical) + main_splitter.addWidget(top_splitter) + main_splitter.addWidget(self.log_panel) + main_splitter.setSizes([600, 200]) + + self.setCentralWidget(main_splitter) + + # Dagster instance + self.instance = DagsterInstance.ephemeral() + self.current_run_id = None + + # Initial log + self.log("Radiuma Workflow Editor started.", "INFO") + self.log("Custom nodes registered with alias: identifier + ClassName.", "INFO") + + def log(self, message, level="INFO"): + fmt = QTextCharFormat() + if level == "INFO": + fmt.setForeground(QColor("green")) + elif level == "WARNING": + fmt.setForeground(QColor("orange")) + elif level == "ERROR": + fmt.setForeground(QColor("red")) + else: + fmt.setForeground(QColor("black")) + + timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + cursor = self.log_panel.textCursor() + cursor.movePosition(QTextCursor.End) + cursor.insertText(f"[{timestamp}] [{level}] {message}\n", fmt) + self.log_panel.setTextCursor(cursor) + + # --- کنترل اجرای Dagster --- + import yaml + + def start_workflow(self): + # Log start of workflow from GUI + self.log("[GUI] Start → Dagster job starting...", "INFO") + + try: + # Load configuration from configs.yaml + with open("configs.yaml", "r", encoding="utf-8") as f: + run_config = yaml.safe_load(f) + + # Check which job type is defined in the config + if "radiomics_batch_job" in run_config: + # Batch mode → run multiple cases automatically + self.log("[GUI] Batch mode detected → Running Batch Workflow", "INFO") + result = radiomics_batch_job.execute_in_process( + run_config=run_config["radiomics_batch_job"], + instance=self.instance + ) + elif "radiomics_job" in run_config: + # Single case mode → run one case only + self.log("[GUI] Single case detected → Running Single Workflow", "INFO") + result = radiomics_job.execute_in_process( + run_config=run_config["radiomics_job"], + instance=self.instance + ) + else: + # Config file does not contain required job section + self.log("[GUI] Config file missing required job section.", "ERROR") + return + + # Save run_id and log success/failure + self.current_run_id = result.run_id + self.log(f"[GUI] Workflow finished. run_id={self.current_run_id} Success={result.success}", "INFO") + + except Exception as e: + # Log any unexpected errors + self.log(f"[GUI] Workflow error: {e}", "ERROR") + + def stop_workflow(self): + if self.current_run_id: + self.instance.cancel_run(self.current_run_id) + self.log(f"[GUI] Workflow stopped. run_id={self.current_run_id}", "WARNING") + + def resume_workflow(self): + if self.current_run_id: + self.instance.resume_run(self.current_run_id) + self.log(f"[GUI] Workflow resumed. run_id={self.current_run_id}", "INFO") + + def cancel_workflow(self): + if self.current_run_id: + self.instance.cancel_run(self.current_run_id) + self.log(f"[GUI] Workflow cancelled. run_id={self.current_run_id}", "ERROR") + + +def main(): + app = QApplication(sys.argv) + window = VisualEditor() + window.show() + sys.exit(app.exec()) + + +if __name__ == '__main__': + main() diff --git a/Dagster/Dagster_Op/app/main.py b/Dagster/Dagster_Op/app/main.py new file mode 100644 index 0000000..aab38fa --- /dev/null +++ b/Dagster/Dagster_Op/app/main.py @@ -0,0 +1,42 @@ +import yaml +from dagster_radiuma.jobs import radiomics_job, radiomics_batch_job +import app.gui.visual_editor as visual_editor + + +def run_radiomics_pipeline(config_file: str): + # Load YAML config + with open(config_file, "r", encoding="utf-8") as f: + run_config = yaml.safe_load(f) + + # Check which job to run + if "radiomics_batch_job" in run_config: + print("[INFO] Batch mode detected → Running Batch Workflow") + result = radiomics_batch_job.execute_in_process( + run_config=run_config["radiomics_batch_job"] + ) + print(f"[RESULT] Batch workflow success = {result.success}") + + elif "radiomics_job" in run_config: + print("[INFO] Single case detected → Running Single Workflow") + result = radiomics_job.execute_in_process( + run_config=run_config["radiomics_job"] + ) + print(f"[RESULT] Single workflow success = {result.success}") + + else: + raise ValueError( + "Config file must contain either 'radiomics_job' or 'radiomics_batch_job' section." + ) + + +if __name__ == "__main__": + print("[INFO] Radiuma Workflow Editor started.") + + use_gui = True # + + if use_gui: + + visual_editor.main() + else: + + run_radiomics_pipeline("configs.yaml") diff --git a/Dagster/Dagster_Op/app/nodes/__pycache__/build.cpython-311.pyc b/Dagster/Dagster_Op/app/nodes/__pycache__/build.cpython-311.pyc new file mode 100644 index 0000000..7d26253 Binary files /dev/null and b/Dagster/Dagster_Op/app/nodes/__pycache__/build.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/nodes/__pycache__/imaging.cpython-311.pyc b/Dagster/Dagster_Op/app/nodes/__pycache__/imaging.cpython-311.pyc new file mode 100644 index 0000000..201c8e1 Binary files /dev/null and b/Dagster/Dagster_Op/app/nodes/__pycache__/imaging.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/app/nodes/build.py b/Dagster/Dagster_Op/app/nodes/build.py new file mode 100644 index 0000000..db9a159 --- /dev/null +++ b/Dagster/Dagster_Op/app/nodes/build.py @@ -0,0 +1,27 @@ +from app.engine.core import Node +from app.nodes.imaging import image_reader_fn, image_registration_fn, image_filter_fn, image_writer_fn + +def build_nodes(): + return [ + Node("image_reader", image_reader_fn, inputs=[], outputs=["image_path"]), + Node("image_registration", image_registration_fn, inputs=["image_path"], outputs=["registered_path"]), + Node("image_filter", image_filter_fn, inputs=["registered_path"], outputs=["filtered_path"]), + Node("image_writer", image_writer_fn, inputs=["filtered_path"], outputs=["final_output_path"]), + ] + + + + + +# from app.engine.core import Node +# from app.nodes.imaging import image_reader_fn, image_registration_fn, image_filter_fn, image_writer_fn +# from app.features.pysera_node import pysera_extract_fn + +# def build_nodes(): +# return [ +# Node("image_reader", image_reader_fn, [], ["image_path"]), +# Node("image_registration", image_registration_fn, ["image_path"], ["registered_path"]), +# Node("image_filter", image_filter_fn, ["registered_path"], ["filtered_path"]), +# Node("pysera_extract", pysera_extract_fn, ["filtered_path"], ["features_path"]), +# Node("image_writer", image_writer_fn, ["filtered_path"], ["final_output_path"]), +# ] \ No newline at end of file diff --git a/Dagster/Dagster_Op/app/nodes/imaging.py b/Dagster/Dagster_Op/app/nodes/imaging.py new file mode 100644 index 0000000..52ba473 --- /dev/null +++ b/Dagster/Dagster_Op/app/nodes/imaging.py @@ -0,0 +1,54 @@ +from pathlib import Path +import numpy as np +import SimpleITK as sitk + +ARTIFACTS = Path("app/storage/artifacts") +ARTIFACTS.mkdir(parents=True, exist_ok=True) + +def image_reader_fn(state): + # Decide input file (DICOM series or NIfTI) + input_image_path = Path("data/images/CT_AVM.nii.gz") # adapt as needed + img = sitk.ReadImage(str(input_image_path)) + out_path = ARTIFACTS / "image_reader.nii.gz" + sitk.WriteImage(img, str(out_path)) + return {"image_path": str(out_path)} + +def image_registration_fn(state): + fixed_path = state.get_outputs("image_reader").get("image_path") + moving_path = fixed_path # demo: register image to itself; replace with real moving image + fixed = sitk.ReadImage(fixed_path) + moving = sitk.ReadImage(moving_path) + + # Simple rigid registration (demo) + registration_method = sitk.ImageRegistrationMethod() + registration_method.SetMetricAsMeanSquares() + registration_method.SetOptimizerAsRegularStepGradientDescent( + learningRate=1.0, minStep=1e-4, numberOfIterations=100 + ) + registration_method.SetInterpolator(sitk.sitkLinear) + initial_transform = sitk.CenteredTransformInitializer( + fixed, moving, sitk.Euler3DTransform(), sitk.CenteredTransformInitializerFilter.GEOMETRY + ) + registration_method.SetInitialTransform(initial_transform, inPlace=False) + final_transform = registration_method.Execute(fixed, moving) + + resampled = sitk.Resample(moving, fixed, final_transform, sitk.sitkLinear, 0.0, moving.GetPixelID()) + out_path = ARTIFACTS / "registered.nii.gz" + sitk.WriteImage(resampled, str(out_path)) + return {"registered_path": str(out_path), "transform": "euler3d"} + +def image_filter_fn(state): + reg_path = state.get_outputs("image_registration").get("registered_path") + img = sitk.ReadImage(reg_path) + # Example filter: Gaussian smoothing + filtered = sitk.DiscreteGaussian(img, variance=1.5) + out_path = ARTIFACTS / "filtered.nii.gz" + sitk.WriteImage(filtered, str(out_path)) + return {"filtered_path": str(out_path)} + +def image_writer_fn(state): + # Writer here is just saving; already done by previous steps; simulate final export + filtered_path = state.get_outputs("image_filter").get("filtered_path") + final_path = ARTIFACTS / "final_output.nii.gz" + Path(final_path).write_bytes(Path(filtered_path).read_bytes()) + return {"final_output_path": str(final_path)} diff --git a/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_asset_features.json b/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_asset_features.json new file mode 100644 index 0000000..b9ccbce --- /dev/null +++ b/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_asset_features.json @@ -0,0 +1,13 @@ +{ + "shape": [ + 58, + 248, + 175 + ], + "spacing": [ + 0.8125, + 0.8125, + 2.3970494270324707 + ], + "intensity_sum": 95678796.0 +} \ No newline at end of file diff --git a/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_features.json b/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_features.json new file mode 100644 index 0000000..66e1416 --- /dev/null +++ b/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_features.json @@ -0,0 +1,9 @@ +{ + "voxel_spacing": [ + 0.8125, + 0.8125, + 2.3970494270324707 + ], + "mask_voxels": 2517200, + "mask_border_voxels": 0 +} \ No newline at end of file diff --git a/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_fused.nii.gz b/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_fused.nii.gz new file mode 100644 index 0000000..cbce671 Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_fused.nii.gz differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_registered.nii.gz b/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_registered.nii.gz new file mode 100644 index 0000000..1bec252 Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/CT_pitch_registered.nii.gz differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/features b/Dagster/Dagster_Op/app/storage/artifacts/features new file mode 100644 index 0000000..2f2b348 Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/features differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/filtered.nii.gz b/Dagster/Dagster_Op/app/storage/artifacts/filtered.nii.gz new file mode 100644 index 0000000..7f44dd7 Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/filtered.nii.gz differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/final_output.nii.gz b/Dagster/Dagster_Op/app/storage/artifacts/final_output.nii.gz new file mode 100644 index 0000000..7f44dd7 Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/final_output.nii.gz differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/fused_image b/Dagster/Dagster_Op/app/storage/artifacts/fused_image new file mode 100644 index 0000000..23ae615 Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/fused_image differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/image_reader.nii.gz b/Dagster/Dagster_Op/app/storage/artifacts/image_reader.nii.gz new file mode 100644 index 0000000..ce0bf78 Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/image_reader.nii.gz differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/raw_image b/Dagster/Dagster_Op/app/storage/artifacts/raw_image new file mode 100644 index 0000000..5aa520f Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/raw_image differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/raw_mask b/Dagster/Dagster_Op/app/storage/artifacts/raw_mask new file mode 100644 index 0000000..7d77269 Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/raw_mask differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/registered.nii.gz b/Dagster/Dagster_Op/app/storage/artifacts/registered.nii.gz new file mode 100644 index 0000000..5ea6f8f Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/registered.nii.gz differ diff --git a/Dagster/Dagster_Op/app/storage/artifacts/registered_image b/Dagster/Dagster_Op/app/storage/artifacts/registered_image new file mode 100644 index 0000000..b57bb4a Binary files /dev/null and b/Dagster/Dagster_Op/app/storage/artifacts/registered_image differ diff --git a/Dagster/Dagster_Op/app/storage/state/state.json b/Dagster/Dagster_Op/app/storage/state/state.json new file mode 100644 index 0000000..c283203 --- /dev/null +++ b/Dagster/Dagster_Op/app/storage/state/state.json @@ -0,0 +1,30 @@ +{ + "steps": { + "image_reader": { + "status": "success", + "outputs": { + "image_path": "app\\storage\\artifacts\\image_reader.nii.gz" + } + }, + "image_registration": { + "status": "success", + "outputs": { + "registered_path": "app\\storage\\artifacts\\registered.nii.gz", + "transform": "euler3d" + } + }, + "image_filter": { + "status": "success", + "outputs": { + "filtered_path": "app\\storage\\artifacts\\filtered.nii.gz" + } + }, + "image_writer": { + "status": "success", + "outputs": { + "final_output_path": "app\\storage\\artifacts\\final_output.nii.gz" + } + } + }, + "cancelled": true +} \ No newline at end of file diff --git a/Dagster/Dagster_Op/configs.yaml b/Dagster/Dagster_Op/configs.yaml new file mode 100644 index 0000000..0c67adf --- /dev/null +++ b/Dagster/Dagster_Op/configs.yaml @@ -0,0 +1,28 @@ +radiomics_batch_job: + ops: + enumerate_cases_auto: + config: + images_dir: "C:/Users/Omen16/Documents/Radiuma_Mini/data/images" + masks_dir: "C:/Users/Omen16/Documents/Radiuma_Mini/data/masks" + image_exts: [".nii.gz"] + mask_exts: [".nii.gz"] + mask_suffixes: ["_mask.nii.gz"] + allow_fuzzy_match: true + fuzzy_max_distance: 5 + verbose: true + +radiomics_job: + ops: + read_image_and_mask: + config: + image_path: "C:/Users/Omen16/Documents/Radiuma_Mini/data/images/CT_AVM.nii.gz" + mask_path: "C:/Users/Omen16/Documents/Radiuma_Mini/data/masks/CT_AVM_mask.nii.gz" + filter_image: + config: + sigma: 2.0 + extract_features: + config: {} + write_outputs: + config: + image_out: "dagster_out_single.nii.gz" + features_out: "dagster_feats_single.csv" diff --git a/Dagster/Dagster_Op/dagster_radiuma/__init__.py b/Dagster/Dagster_Op/dagster_radiuma/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Dagster/Dagster_Op/dagster_radiuma/__pycache__/__init__.cpython-311.pyc b/Dagster/Dagster_Op/dagster_radiuma/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..441e4e9 Binary files /dev/null and b/Dagster/Dagster_Op/dagster_radiuma/__pycache__/__init__.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/dagster_radiuma/__pycache__/jobs.cpython-311.pyc b/Dagster/Dagster_Op/dagster_radiuma/__pycache__/jobs.cpython-311.pyc new file mode 100644 index 0000000..d811211 Binary files /dev/null and b/Dagster/Dagster_Op/dagster_radiuma/__pycache__/jobs.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/dagster_radiuma/__pycache__/ops.cpython-311.pyc b/Dagster/Dagster_Op/dagster_radiuma/__pycache__/ops.cpython-311.pyc new file mode 100644 index 0000000..4e0accb Binary files /dev/null and b/Dagster/Dagster_Op/dagster_radiuma/__pycache__/ops.cpython-311.pyc differ diff --git a/Dagster/Dagster_Op/dagster_radiuma/jobs.py b/Dagster/Dagster_Op/dagster_radiuma/jobs.py new file mode 100644 index 0000000..79c7ac9 --- /dev/null +++ b/Dagster/Dagster_Op/dagster_radiuma/jobs.py @@ -0,0 +1,104 @@ +from dagster import job, graph, op +from pathlib import Path +from .ops import ( + read_image_and_mask, + read_image_and_mask_from_case, + filter_image, + extract_features, + write_outputs, + enumerate_cases_auto +) + +# Extract image file name from config for single file mode +@op(config_schema={"image_path": str}) +def get_case_name_from_config(context) -> str: + image_path = context.op_config["image_path"] + return Path(image_path).stem + +# Extract case_name from case dictionary for batch mode +@op +def get_case_name_from_case(context, case: dict) -> str: + return case["case_name"] + +# Single-case workflow job +@job(config={ + "ops": { + "read_image_and_mask": { + "config": { + "image_path": "C:/Users/Omen16/Documents/Radiuma_Mini/data/images/CT_pitch.nii.gz", + "mask_path": "C:/Users/Omen16/Documents/Radiuma_Mini/data/masks/CT_pitch_mask.nii.gz", + } + }, + "get_case_name_from_config": { + "config": { + "image_path": "C:/Users/Omen16/Documents/Radiuma_Mini/data/images/CT_pitch.nii.gz" + } + } + } +}) +def radiomics_job(): + image, mask = read_image_and_mask() + filtered = filter_image(image) + feats = extract_features(image=filtered, mask=mask) + case_name = get_case_name_from_config() + write_outputs(features=feats, case_name=case_name) + +# Graph that processes one case dict (used for batch mapping) +@graph +def process_case(case): + image, mask = read_image_and_mask_from_case(case) + filtered = filter_image(image) + feats = extract_features(image=filtered, mask=mask) + case_name = get_case_name_from_case(case) + write_outputs(features=feats, case_name=case_name) + +# Batch workflow job (auto-discovery + dynamic mapping) +@job +def radiomics_batch_job(): + cases = enumerate_cases_auto() + cases.map(process_case) + +# Preprocessing-only workflow job +@job(config={ + "ops": { + "read_image_and_mask": { + "config": { + "image_path": "C:/Users/Omen16/Documents/Radiuma_Mini/data/images/CT_pitch.nii.gz", + "mask_path": "C:/Users/Omen16/Documents/Radiuma_Mini/data/masks/CT_pitch_mask.nii.gz", + } + }, + "get_case_name_from_config": { + "config": { + "image_path": "C:/Users/Omen16/Documents/Radiuma_Mini/data/images/CT_pitch.nii.gz" + } + } + } +}) +def preprocessing_job(): + image, mask = read_image_and_mask() + filtered = filter_image(image) + feats = extract_features(image=filtered, mask=mask) + case_name = get_case_name_from_config() + write_outputs(features=feats, case_name=case_name) + +# Feature-extraction-only workflow job +@job(config={ + "ops": { + "read_image_and_mask": { + "config": { + "image_path": "C:/Users/Omen16/Documents/Radiuma_Mini/data/images/CT_pitch.nii.gz", + "mask_path": "C:/Users/Omen16/Documents/Radiuma_Mini/data/masks/CT_pitch_mask.nii.gz", + } + }, + "get_case_name_from_config": { + "config": { + "image_path": "C:/Users/Omen16/Documents/Radiuma_Mini/data/images/CT_pitch.nii.gz" + } + } + } +}) +def feature_extraction_job(): + image, mask = read_image_and_mask() + feats = extract_features(image=image, mask=mask) + case_name = get_case_name_from_config() + write_outputs(features=feats, case_name=case_name) \ No newline at end of file diff --git a/Dagster/Dagster_Op/dagster_radiuma/ops.py b/Dagster/Dagster_Op/dagster_radiuma/ops.py new file mode 100644 index 0000000..0094f36 --- /dev/null +++ b/Dagster/Dagster_Op/dagster_radiuma/ops.py @@ -0,0 +1,223 @@ +import os +import numpy as np +import pandas as pd +from pathlib import Path +import SimpleITK as sitk +from dagster import op, In, Out, DynamicOut, DynamicOutput + + +# Utility functions +def remove_ext(filename: str, exts): + fname = filename + for ext in exts: + if fname.lower().endswith(ext.lower()): + return fname[:-len(ext)] + return fname + +def strip_suffixes(name_no_ext: str, suffixes): + for s in suffixes: + if name_no_ext.endswith(s): + return name_no_ext[:-len(s)] + return name_no_ext + +def levenshtein(a: str, b: str) -> int: + if a == b: + return 0 + m, n = len(a), len(b) + dp = list(range(n + 1)) + for i in range(1, m + 1): + prev, dp[0] = dp[0], i + for j in range(1, n + 1): + cur = dp[j] + cost = 0 if a[i - 1] == b[j - 1] else 1 + dp[j] = min(dp[j] + 1, dp[j - 1] + 1, prev + cost) + prev = cur + return dp[n] + +def list_files(root: Path, exts): + files = [] + for p in Path(root).rglob("*"): + if p.is_file() and any(str(p).lower().endswith(ext.lower()) for ext in exts): + files.append(p) + return sorted(files) + +# Single-case readers + +@op( + config_schema={ + "image_path": str, + "mask_path": str, + }, + out={"image": Out(), "mask": Out()} +) +def read_image_and_mask(context): + image_path = context.op_config["image_path"] + mask_path = context.op_config["mask_path"] + + if not os.path.exists(image_path): + raise FileNotFoundError(f"Image not found: {image_path}") + if not os.path.exists(mask_path): + raise FileNotFoundError(f"Mask not found: {mask_path}") + + image = sitk.ReadImage(image_path) + mask = sitk.ReadImage(mask_path) + img_arr = sitk.GetArrayFromImage(image) + msk_arr = sitk.GetArrayFromImage(mask) + + context.log.info(f"[single] image:{img_arr.shape} mask:{msk_arr.shape}") + return img_arr, msk_arr + +@op( + ins={"case": In()}, + out={"image": Out(), "mask": Out()} +) +def read_image_and_mask_from_case(context, case: dict): + image_path = case["image_path"] + mask_path = case["mask_path"] + + if not os.path.exists(image_path): + raise FileNotFoundError(f"Image not found: {image_path}") + if not os.path.exists(mask_path): + raise FileNotFoundError(f"Mask not found: {mask_path}") + + image = sitk.ReadImage(image_path) + mask = sitk.ReadImage(mask_path) + img_arr = sitk.GetArrayFromImage(image) + msk_arr = sitk.GetArrayFromImage(mask) + + context.log.info(f"[case] image:{img_arr.shape} mask:{msk_arr.shape}") + return img_arr, msk_arr + + +# Preprocess, features, writer + +@op(out=Out()) +def filter_image(context, image: np.ndarray): + context.log.info(f"[filter] image:{image.shape}") + return image + +@op(ins={"image": In(), "mask": In()}, out=Out()) +def extract_features(context, image: np.ndarray, mask: np.ndarray): + masked = image[mask > 0] + mean_val = float(masked.mean()) if masked.size > 0 else 0.0 + features = {"mean_intensity": mean_val} + context.log.info(f"[features] {features}") + return features + + +RESULTS_DIR = Path("results") +RESULTS_DIR.mkdir(exist_ok=True) + +@op(ins={"features": In(), "case_name": In()}, out=Out()) +def write_outputs(context, features: dict, case_name: str): + # Define output CSV file path + csv_path = RESULTS_DIR / "features.csv" + + # Prepare row with case name and features + row = {"case_name": case_name} + row.update(features) + + # If CSV does not exist, create new file with header + if not csv_path.exists(): + df = pd.DataFrame([row]) + df.to_csv(csv_path, index=False) + else: + # Append new row to existing CSV + df = pd.DataFrame([row]) + df.to_csv(csv_path, mode="a", header=False, index=False) + + context.log.info(f"[write] features appended to {csv_path}") + + # Optional Excel export (currently disabled) + # excel_path = RESULTS_DIR / "features.xlsx" + # df.to_excel(excel_path, index=False) + + return str(csv_path) + + +# Batch case discovery + +@op( + config_schema={ + "images_dir": str, + "masks_dir": str, + "image_exts": list, + "mask_exts": list, + "mask_suffixes": list, + "allow_fuzzy_match": bool, + "fuzzy_max_distance": int, + "verbose": bool, + }, + out=DynamicOut() +) +def enumerate_cases_auto(context): + cfg = context.op_config + images_dir = Path(cfg["images_dir"]).expanduser().resolve() + masks_dir = Path(cfg["masks_dir"]).expanduser().resolve() + image_exts = [ext.lower() for ext in cfg["image_exts"]] + mask_exts = [ext.lower() for ext in cfg["mask_exts"]] + mask_suffixes = cfg["mask_suffixes"] + allow_fuzzy = cfg["allow_fuzzy_match"] + max_dist = int(cfg["fuzzy_max_distance"]) + verbose = bool(cfg.get("verbose", True)) + + img_files = list_files(images_dir, image_exts) + msk_files = list_files(masks_dir, mask_exts) + + context.log.info(f"[discover] images_dir={images_dir} masks_dir={masks_dir}") + context.log.info(f"[discover] found images: {len(img_files)}") + for p in img_files: + context.log.info(f" - image: {p.name}") + context.log.info(f"[discover] found masks: {len(msk_files)}") + for p in msk_files: + context.log.info(f" - mask: {p.name}") + + msk_index = {} + for m in msk_files: + base = strip_suffixes(remove_ext(m.name, mask_exts), mask_suffixes).lower() + msk_index.setdefault(base, []).append(m) + + yielded = 0 + skipped = 0 + + for img in img_files: + base = remove_ext(img.name, image_exts).lower() + candidates = msk_index.get(base, []) + chosen = None + reason = None + + if candidates: + chosen = candidates[0] + else: + if allow_fuzzy and msk_files: + best = None + best_d = 10**9 + for m in msk_files: + mbase = strip_suffixes(remove_ext(m.name, mask_exts), mask_suffixes).lower() + d = levenshtein(base, mbase) + if d < best_d: + best, best_d = m, d + if best is not None and best_d <= max_dist: + chosen = best + else: + reason = f"no exact or fuzzy match (best distance {best_d})" + + if chosen is None: + skipped += 1 + context.log.warning(f"[skip] image:{img.name} → no mask found ({reason}) | base(image)='{base}'") + continue + + case = { + "image_path": str(img), + "mask_path": str(chosen), + "case_name": remove_ext(img.name, image_exts), + } + context.log.info(f"[pair] {case['case_name']} → image:{img.name} mask:{Path(chosen).name}") + yield DynamicOutput(case, mapping_key=case["case_name"]) + yielded += 1 + + context.log.info(f"[summary] yielded={yielded} skipped={skipped}") + + if yielded == 0: + context.log.error("No valid cases discovered. See [skip] messages above for reasons.") + raise Exception("No valid cases discovered. See logs for details.") diff --git a/Dagster/Dagster_Op/data/images/CT_pitch.nii.gz b/Dagster/Dagster_Op/data/images/CT_pitch.nii.gz new file mode 100644 index 0000000..1a51792 Binary files /dev/null and b/Dagster/Dagster_Op/data/images/CT_pitch.nii.gz differ diff --git a/Dagster/Dagster_Op/data/masks/CT_pitch_mask.nii.gz b/Dagster/Dagster_Op/data/masks/CT_pitch_mask.nii.gz new file mode 100644 index 0000000..b7a2ff1 Binary files /dev/null and b/Dagster/Dagster_Op/data/masks/CT_pitch_mask.nii.gz differ diff --git a/Dagster/Dagster_Op/radiuma_workflow.py b/Dagster/Dagster_Op/radiuma_workflow.py new file mode 100644 index 0000000..0e7767c --- /dev/null +++ b/Dagster/Dagster_Op/radiuma_workflow.py @@ -0,0 +1,65 @@ +from dagster import job, op, In +import SimpleITK as sitk +from pathlib import Path +from app.features.pysera_node import run_pysera +# Write Your Codes Here: + + +ARTIFACTS_DIR = Path("artifacts") +ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) + +@op(config_schema={"image_file": str}) +def image_reader(context): + # The Image File Is Read From Config. + image_file = context.op_config["image_file"] + input_path = Path("data/images") / image_file + img = sitk.ReadImage(str(input_path)) + out_path = ARTIFACTS_DIR / "image_reader.nii.gz" + sitk.WriteImage(img, str(out_path)) + return str(out_path) + +@op(config_schema={"mask_file": str}) +def mask_reader(context): + mask_file = context.op_config["mask_file"] + mask_path = Path("data/masks") / mask_file + mask = sitk.ReadImage(str(mask_path)) + out_path = ARTIFACTS_DIR / "mask_reader.nii.gz" + sitk.WriteImage(mask, str(out_path)) + return str(out_path) + +@op +def image_registration(image_path: str): + img = sitk.ReadImage(image_path) + out_path = ARTIFACTS_DIR / "registered.nii.gz" + sitk.WriteImage(img, str(out_path)) + return str(out_path) + +@op +def image_filter(registered_path: str): + img = sitk.ReadImage(registered_path) + filtered = sitk.DiscreteGaussian(img, variance=1.5) + out_path = ARTIFACTS_DIR / "filtered.nii.gz" + sitk.WriteImage(filtered, str(out_path)) + return str(out_path) + +@op +def pysera_extract(filtered_path: str, mask_path: str): + features = run_pysera(filtered_path, mask_path) + out_path = ARTIFACTS_DIR / "features.json" + out_path.write_text(str(features)) + return str(out_path) + +@op +def image_writer(filtered_path: str): + final_path = ARTIFACTS_DIR / "final_output.nii.gz" + final_path.write_bytes(Path(filtered_path).read_bytes()) + return str(final_path) + +@job +def radiuma_job(): + img = image_reader() + mask = mask_reader() + reg = image_registration(img) + filt = image_filter(reg) + pysera_extract(filt, mask) + image_writer(filt) diff --git a/Dagster/Dagster_Op/requirements.txt b/Dagster/Dagster_Op/requirements.txt new file mode 100644 index 0000000..51b9fe5 --- /dev/null +++ b/Dagster/Dagster_Op/requirements.txt @@ -0,0 +1,211 @@ +aiohappyeyeballs==2.6.1 +aiohttp==3.13.2 +aiosignal==1.4.0 +alabaster==1.0.0 +alembic==1.17.2 +annotated-types==0.7.0 +antlr4-python3-runtime==4.13.2 +anyio==4.11.0 +app-model==0.4.0 +appdirs==1.4.4 +asttokens==3.0.1 +attrs==25.4.0 +babel==2.17.0 +backoff==2.2.1 +bermuda==0.1.6 +build==1.3.0 +cachey==0.2.1 +certifi==2025.11.12 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +colorama==0.4.6 +coloredlogs==14.0 +comm==0.2.3 +connected-components-3d==3.26.1 +dagit==1.12.3 +dagster==1.12.3 +dagster-graphql==1.12.3 +dagster-pandas==0.28.3 +dagster-pipes==1.12.3 +dagster-shell==0.11.14 +dagster-webserver==1.12.3 +dagster_shared==1.12.3 +dask==2025.11.0 +dataclasses==0.6 +debugpy==1.8.17 +decorator==5.2.1 +docstring_parser==0.17.0 +docutils==0.21.2 +donfig==0.8.1.post1 +et_xmlfile==2.0.0 +executing==2.2.1 +filelock==3.20.0 +flexcache==0.3 +flexparser==0.4 +freetype-py==2.5.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +google-crc32c==1.7.1 +gql==3.5.3 +graphene==3.4.3 +graphql-core==3.2.6 +graphql-relay==3.2.0 +greenlet==3.2.4 +grpcio==1.76.0 +grpcio-health-checking==1.76.0 +h11==0.16.0 +HeapDict==1.0.1 +hsluv==5.0.4 +httptools==0.7.1 +humanfriendly==10.0 +idna==3.11 +ImageIO==2.37.2 +imagesize==1.4.1 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +in-n-out==0.2.1 +ipykernel==6.31.0 +ipython==9.7.0 +ipython_pygments_lexers==1.1.1 +itk==5.4.5 +itk-core==5.4.5 +itk-filtering==5.4.5 +itk-io==5.4.5 +itk-numerics==5.4.5 +itk-registration==5.4.5 +itk-segmentation==5.4.5 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.2 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +jupyter_client==8.6.3 +jupyter_core==5.9.1 +kiwisolver==1.4.9 +lazy_loader==0.4 +llvmlite==0.45.1 +locket==1.0.0 +loguru==0.7.3 +magicgui==0.10.1 +Mako==1.3.10 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +matplotlib-inline==0.2.1 +mdurl==0.1.2 +multidict==6.7.0 +napari==0.6.6 +napari-console==0.1.4 +napari-plugin-engine==0.2.0 +napari-plugin-manager==0.1.8 +napari-svg==0.2.1 +nest-asyncio==1.6.0 +networkx==3.6 +nibabel==5.3.2 +NodeGraphQt==0.6.43 +npe2==0.7.9 +numba==0.62.1 +numcodecs==0.16.5 +numpy==2.2.6 +numpydoc==1.9.0 +opencv-python==4.12.0.88 +openpyxl==3.1.5 +packaging==25.0 +pandas==2.3.3 +parso==0.8.5 +partd==1.4.2 +pathlib_abc==0.5.2 +pillow==12.0.0 +Pint==0.25.2 +platformdirs==4.5.0 +pooch==1.8.2 +prompt_toolkit==3.0.52 +propcache==0.4.1 +protobuf==6.33.1 +psutil==7.1.3 +psygnal==0.15.0 +pure_eval==0.2.3 +pyconify==0.2.1 +pydantic==2.12.4 +pydantic-compat==0.1.2 +pydantic_core==2.41.5 +pydicom==3.0.1 +Pygments==2.19.2 +pynrrd==1.1.3 +PyOpenGL==3.1.10 +pyproject_hooks==1.2.0 +PyQt5==5.15.11 +PyQt5-Qt5==5.15.2 +PyQt5_sip==12.17.1 +pyreadline3==3.5.4 +pysera==2.1.5 +PySide6==6.10.1 +PySide6_Addons==6.10.1 +PySide6_Essentials==6.10.1 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +pytz==2025.2 +pywin32==311 +PyYAML==6.0.3 +pyzmq==27.1.0 +Qt.py==1.4.8 +qtconsole==5.7.0 +QtPy==2.4.3 +referencing==0.37.0 +requests==2.32.5 +requests-toolbelt==1.0.0 +rich==14.2.0 +roman-numerals-py==3.1.0 +rpds-py==0.29.0 +rt-utils==1.2.7 +scikit-image==0.25.2 +scikit-learn==1.7.2 +scipy==1.16.3 +shellingham==1.5.4 +shiboken6==6.10.1 +simpleitk==2.5.3 +six==1.17.0 +sniffio==1.3.1 +snowballstemmer==3.0.1 +Sphinx==8.2.3 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 +SQLAlchemy==2.0.44 +stack-data==0.6.3 +starlette==0.50.0 +structlog==25.5.0 +superqt==0.7.6 +tabulate==0.9.0 +threadpoolctl==3.6.0 +tifffile==2025.10.16 +tomli==2.3.0 +tomli_w==1.2.0 +tomlkit==0.13.3 +toolz==1.1.0 +toposort==1.10 +tornado==6.5.2 +tqdm==4.67.1 +traitlets==5.14.3 +triangle==20250106 +typer==0.20.0 +types-PySide2==5.15.2.1.8 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2025.2 +universal_pathlib==0.3.6 +urllib3==2.5.0 +uvicorn==0.38.0 +vispy==0.15.2 +watchdog==6.0.0 +watchfiles==1.1.1 +wcwidth==0.2.14 +websockets==15.0.1 +win32_setctime==1.2.0 +wrapt==2.0.1 +yarl==1.22.0 +zarr==3.1.5 +zipp==3.23.0 diff --git a/Dagster/Dagster_Op/run.bat b/Dagster/Dagster_Op/run.bat new file mode 100644 index 0000000..9bcd406 --- /dev/null +++ b/Dagster/Dagster_Op/run.bat @@ -0,0 +1,9 @@ +@echo off +REM Activate virtual environment +call venv\Scripts\activate +cd /d C:\Users\Omen16\Documents\Radiuma_Mini + +REM Run the Radiuma Mini application +python -m app.main + +pause diff --git a/Dagster/Radiuma_API/README.md b/Dagster/Radiuma_API/README.md new file mode 100644 index 0000000..6597905 --- /dev/null +++ b/Dagster/Radiuma_API/README.md @@ -0,0 +1,8 @@ +Description: +# In this project, using Pytest, the API layer codes of the Radioma Desktop Ver 2.0 project were tested with 20 tests. + +Dependencies: +# pip install pytest + +Running: +# pytest test rnd/run_rnd_workflow \ No newline at end of file diff --git a/Dagster/Radiuma_API/radiuma_api/__init__.py b/Dagster/Radiuma_API/radiuma_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Dagster/Radiuma_API/radiuma_api/__pycache__/__init__.cpython-313.pyc b/Dagster/Radiuma_API/radiuma_api/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..938e603 Binary files /dev/null and b/Dagster/Radiuma_API/radiuma_api/__pycache__/__init__.cpython-313.pyc differ diff --git a/Dagster/Radiuma_API/radiuma_api/__pycache__/asset.cpython-313.pyc b/Dagster/Radiuma_API/radiuma_api/__pycache__/asset.cpython-313.pyc new file mode 100644 index 0000000..286845c Binary files /dev/null and b/Dagster/Radiuma_API/radiuma_api/__pycache__/asset.cpython-313.pyc differ diff --git a/Dagster/Radiuma_API/radiuma_api/__pycache__/execution_context.cpython-313.pyc b/Dagster/Radiuma_API/radiuma_api/__pycache__/execution_context.cpython-313.pyc new file mode 100644 index 0000000..a62f97f Binary files /dev/null and b/Dagster/Radiuma_API/radiuma_api/__pycache__/execution_context.cpython-313.pyc differ diff --git a/Dagster/Radiuma_API/radiuma_api/__pycache__/io_port.cpython-313.pyc b/Dagster/Radiuma_API/radiuma_api/__pycache__/io_port.cpython-313.pyc new file mode 100644 index 0000000..7b4ddbb Binary files /dev/null and b/Dagster/Radiuma_API/radiuma_api/__pycache__/io_port.cpython-313.pyc differ diff --git a/Dagster/Radiuma_API/radiuma_api/__pycache__/module.cpython-313.pyc b/Dagster/Radiuma_API/radiuma_api/__pycache__/module.cpython-313.pyc new file mode 100644 index 0000000..cfcd4a9 Binary files /dev/null and b/Dagster/Radiuma_API/radiuma_api/__pycache__/module.cpython-313.pyc differ diff --git a/Dagster/Radiuma_API/radiuma_api/__pycache__/scheduler.cpython-313.pyc b/Dagster/Radiuma_API/radiuma_api/__pycache__/scheduler.cpython-313.pyc new file mode 100644 index 0000000..0d79fb6 Binary files /dev/null and b/Dagster/Radiuma_API/radiuma_api/__pycache__/scheduler.cpython-313.pyc differ diff --git a/Dagster/Radiuma_API/radiuma_api/__pycache__/task.cpython-313.pyc b/Dagster/Radiuma_API/radiuma_api/__pycache__/task.cpython-313.pyc new file mode 100644 index 0000000..5cd0dee Binary files /dev/null and b/Dagster/Radiuma_API/radiuma_api/__pycache__/task.cpython-313.pyc differ diff --git a/Dagster/Radiuma_API/radiuma_api/__pycache__/workflow.cpython-313.pyc b/Dagster/Radiuma_API/radiuma_api/__pycache__/workflow.cpython-313.pyc new file mode 100644 index 0000000..b838ada Binary files /dev/null and b/Dagster/Radiuma_API/radiuma_api/__pycache__/workflow.cpython-313.pyc differ diff --git a/Dagster/Radiuma_API/radiuma_api/asset.py b/Dagster/Radiuma_API/radiuma_api/asset.py new file mode 100644 index 0000000..4598144 --- /dev/null +++ b/Dagster/Radiuma_API/radiuma_api/asset.py @@ -0,0 +1,14 @@ +class Asset: + """ + Represents the output of a Task. + """ + def __init__(self, data, type_name: str): + self.data = data + self.type_name = type_name + self.preserved = True + + def dismiss(self): + self.preserved = False + + def __repr__(self): + return f"" diff --git a/Dagster/Radiuma_API/radiuma_api/execution_context.py b/Dagster/Radiuma_API/radiuma_api/execution_context.py new file mode 100644 index 0000000..8564988 --- /dev/null +++ b/Dagster/Radiuma_API/radiuma_api/execution_context.py @@ -0,0 +1,53 @@ +from typing import Dict, Any +from radiuma_api.asset import Asset +from radiuma_api.io_port import InPort +from radiuma_api.io_port import OutPort +# Write Codes Here: + + +class ExecutionContext: + """ + Holds runtime execution data for a Workflow run. + """ + def __init__(self, execution_id: str | None = None): + self.execution_id = execution_id + self._data: Dict[OutPort, Asset] = {} + self._metadata: Dict[str, Any] = {} + + # Output Data Recording + def put(self, output_port: OutPort, asset: Asset): + """Register an asset produced by an OutputPort.""" + self._data[output_port] = asset + + # Getting Data For InputPort + def get(self, input_port: InPort): + """Retrieve the asset connected to this InputPort.""" + if input_port.connected_output is None: + return None + return self._data.get(input_port.connected_output) + + # Check Data Availability + def has_data(self, input_port: InPort): + return self.get(input_port) is not None + + # Metadata management + def set_meta(self, key: str, value: Any): + self._metadata[key] = value + + def get_meta(self, key: str, default=None): + return self._metadata.get(key, default) + + # Status View + def status_view(self): + """Return a lightweight snapshot of current execution state.""" + return { + "execution_id": self.execution_id, + "assets": { + f"{port.parent_task.name}.{port.name}": { + "contract": repr(port.contract), + "asset_type": asset.type_name + } + for port, asset in self._data.items() + }, + "metadata": self._metadata.copy(), + } diff --git a/Dagster/Radiuma_API/radiuma_api/io_port.py b/Dagster/Radiuma_API/radiuma_api/io_port.py new file mode 100644 index 0000000..840f787 --- /dev/null +++ b/Dagster/Radiuma_API/radiuma_api/io_port.py @@ -0,0 +1,200 @@ +from abc import ABC, abstractmethod +from typing import Optional, Dict, Any, List +# Codes Go Below: + + +# Errors: +class CompatibilityException(Exception): + def __init__(self, reason: str): + super().__init__(reason) + self.reason = reason + + +# DType (Semantic Contract): +class DType(ABC): + def __init__(self, metadata: Optional[Dict[str, Any]] = None): + self.metadata = metadata or {} + + @abstractmethod + def can_connect_to(self, other: "DType", conditions: Optional[Dict[str, Any]] = None): + pass + + def __repr__(self): + return f"{self.__class__.__name__}(metadata={self.metadata})" + + +# COMPOSITE TYPE: +class CompositeType(DType): + """ + Represents a semantic bundle of multiple named DTypes. + Example: + CompositeType({ + "sheet1": TableType(...), + "sheet2": TextType(...), + "image": ImageType(...) + }) + """ + def __init__(self, parts: Dict[str, DType], metadata=None): + super().__init__(metadata) + self.parts = parts # Dict[str, DType] + + def can_connect_to(self, other: "DType", conditions=None): + if not isinstance(other, CompositeType): + raise CompatibilityException("Expected CompositeType") + + if self.parts.keys() != other.parts.keys(): + raise CompatibilityException("CompositeType Keys Mismatch") + + for key in self.parts: + self.parts[key].can_connect_to(other.parts[key], conditions) + + def __repr__(self): + return f"CompositeType({self.parts})" + + +# Image Types: +class ImageType(DType): + def __init__(self, modality: Optional[str] = None, metadata=None): + super().__init__(metadata) + self.modality = modality + + def can_connect_to(self, other: DType, conditions=None): + if not isinstance(other, ImageType): + raise CompatibilityException("Target is not an ImageType") + + if self.modality and other.modality and self.modality != other.modality: + raise CompatibilityException( + f"Image modality mismatch: {self.modality} vs {other.modality}" + ) + + +class NIFTIImageType(ImageType): + def __init__(self, metadata=None): + super().__init__(modality="nifti", metadata=metadata) + + def can_connect_to(self, other: DType, conditions=None): + super().can_connect_to(other, conditions) + # NIFTI-specific rules here + + +class DICOMImageType(ImageType): + def __init__(self, metadata=None): + super().__init__(modality="dicom", metadata=metadata) + + def can_connect_to(self, other: DType, conditions=None): + super().can_connect_to(other, conditions) + # DICOM-specific rules here + + +# Table Types: +class TableType(DType): + def __init__(self, columns: List[str], metadata=None): + super().__init__(metadata) + self.columns = columns + + def can_connect_to(self, other: DType, conditions=None): + if not isinstance(other, TableType): + raise CompatibilityException("Target is not a TableType") + + if self.columns != other.columns: + raise CompatibilityException( + f"Table schema mismatch: {self.columns} vs {other.columns}" + ) + + +class CSVTableType(TableType): + def __init__( + self, + columns: List[str], + delimiter: str = ",", + encoding: str = "utf-8", + metadata=None, + ): + super().__init__(columns, metadata) + self.delimiter = delimiter + self.encoding = encoding + + def can_connect_to(self, other: DType, conditions=None): + if not isinstance(other, CSVTableType): + raise CompatibilityException("CSV tables can only connect to CSV tables") + + super().can_connect_to(other, conditions) + + if self.delimiter != other.delimiter: + raise CompatibilityException( + f"CSV delimiter mismatch: {self.delimiter} vs {other.delimiter}" + ) + + +# Text Type: +class TextType(DType): + def __init__( + self, + language: Optional[str] = None, + encoding: Optional[str] = None, + metadata=None, + ): + super().__init__(metadata) + self.language = language + self.encoding = encoding + + def can_connect_to(self, other: DType, conditions=None): + if not isinstance(other, TextType): + raise CompatibilityException("Target is not TextType") + + if self.language and other.language and self.language != other.language: + raise CompatibilityException( + f"Language mismatch: {self.language} vs {other.language}" + ) + + if self.encoding and other.encoding and self.encoding != other.encoding: + raise CompatibilityException( + f"Encoding mismatch: {self.encoding} vs {other.encoding}" + ) + + +# Ports: +class Port(ABC): + def __init__(self, name: str, dtype: DType): + self.name = name + self.dtype = dtype + self.parent_task = None + + def __repr__(self): + return f"{self.__class__.__name__}({self.name}, {self.dtype})" + + +class OutPort(Port): + def __init__(self, name: str, dtype: DType): + super().__init__(name, dtype) + self._connections: List["InPort"] = [] + + @property + def connections(self): + return list(self._connections) + + def connect(self, in_port: "InPort", conditions: Optional[Dict[str, Any]] = None): + if len(self._connections): + raise CompatibilityException("OutPort max connections reached") + + in_port.can_connect_to(self, conditions) + + self._connections.append(in_port) + in_port._connected_output = self + + +class InPort(Port): + def __init__(self, name: str, dtype: DType, required: bool = True): + super().__init__(name, dtype) + self.required = required + self._connected_output: Optional[OutPort] = None + + @property + def connected_output(self): + return self._connected_output + + def is_ready(self): + return self._connected_output is not None + + def can_connect_to(self, out_port: OutPort, conditions: Optional[Dict[str, Any]] = None): + self.dtype.can_connect_to(out_port.dtype, conditions) diff --git a/Dagster/Radiuma_API/radiuma_api/module.py b/Dagster/Radiuma_API/radiuma_api/module.py new file mode 100644 index 0000000..1b72c21 --- /dev/null +++ b/Dagster/Radiuma_API/radiuma_api/module.py @@ -0,0 +1,34 @@ +from radiuma_api.task import Task, Status, TaskEvent +from radiuma_api.asset import Asset +from radiuma_api.execution_context import ExecutionContext +from radiuma_api.io_port import CompatibilityException +# Write Codes Here: + + +class Module(Task): + """Leaf node: atomic executable unit.""" + def run(self, context: ExecutionContext): + if not self.check_inputs_ready(context): + self._set_status(Status.PENDING) + return + + self._set_status(Status.RUNNING) + + try: + consumed_data = [ + port.connected_output.produced_asset.data + for port in self.inputs + if port.is_ready() + ] + for output_port in self.outputs: + asset = Asset( + data=f"{self.name} processed {consumed_data}", + type_name=output_port.type_spec + ) + context.put(output_port, asset) + + self._set_status(Status.COMPLETED) + + except CompatibilityException as e: + self._set_status(Status.FAILED) + self._emit(TaskEvent.ON_ERROR, {"error": e.reason}) diff --git a/Dagster/Radiuma_API/radiuma_api/scheduler.py b/Dagster/Radiuma_API/radiuma_api/scheduler.py new file mode 100644 index 0000000..3326f26 --- /dev/null +++ b/Dagster/Radiuma_API/radiuma_api/scheduler.py @@ -0,0 +1,85 @@ +from typing import Set +from radiuma_api.task import Task, Status +from radiuma_api.execution_context import ExecutionContext +# Write Codes Here: + + +class Scheduler: + """ + DAG-based execution scheduler. + Decides WHEN a Task can run based on port readiness and orchestrates execution. + """ + + def __init__(self): + self._completed: Set[Task] = set() + self._failed: Set[Task] = set() + + def run(self, root: Task, context: ExecutionContext): + """ + Entry point: execute a Workflow or Task graph. + """ + self._execute_task(root, context) + + def _execute_task(self, task: Task, context: ExecutionContext): + """ + Execute a task respecting DAG dependencies. + If 'task' is a Workflow (has children), delegate to _run_workflow. + Otherwise, run the leaf task when inputs are ready. + """ + # Skip finished tasks + if task.status in (Status.COMPLETED, Status.FAILED, Status.STOPPED): + return + + # Workflow (composite) + if hasattr(task, "children"): + self._run_workflow(task, context) + return + + # Leaf Task: must have inputs ready + if not task.check_inputs_ready(context): + # Optional: diagnostic log + print(f"[Scheduler] Task '{task.name}' not ready, skipping for now.") + return + + try: + task._set_status(Status.RUNNING) + task.run(context) + task._set_status(Status.COMPLETED) + self._completed.add(task) + except Exception as e: + task._set_status(Status.FAILED) + self._failed.add(task) + # Optional: diagnostic log + print(f"[Scheduler] Task '{task.name}' failed: {e}") + # Re-raise to surface error if caller needs to handle + raise + + def _run_workflow(self, workflow: Task, context: ExecutionContext): + """ + Execute children respecting dependencies. + Progress until all children are executed or a deadlock is detected. + """ + workflow._set_status(Status.RUNNING) + + remaining = set(workflow.children) + + while remaining: + progress_made = False + + for child in list(remaining): + if child.check_inputs_ready(context): + self._execute_task(child, context) + remaining.remove(child) + progress_made = True + + if not progress_made: + # Deadlock: dependencies not resolvable with current data + raise RuntimeError( + f"Workflow '{workflow.name}' deadlock: unresolved dependencies" + ) + + # Final workflow status based on children results + if any(c.status == Status.FAILED for c in workflow.children): + workflow._set_status(Status.FAILED) + else: + workflow._set_status(Status.COMPLETED) diff --git a/Dagster/Radiuma_API/radiuma_api/task.py b/Dagster/Radiuma_API/radiuma_api/task.py new file mode 100644 index 0000000..25bbadb --- /dev/null +++ b/Dagster/Radiuma_API/radiuma_api/task.py @@ -0,0 +1,83 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import List, Any, Optional +from radiuma_api.io_port import InPort +from radiuma_api.io_port import OutPort +from radiuma_api.execution_context import ExecutionContext +# Write Codes Here: + + +class Status(Enum): + PENDING = "pending" + READY = "ready" + RUNNING = "running" + PAUSED = "paused" + STOPPED = "stopped" + COMPLETED = "completed" + FAILED = "failed" + +class TaskEvent(Enum): + BEFORE_RUN = "before_run" + AFTER_RUN = "after_run" + ON_ERROR = "on_error" + STATUS_CHANGED = "status_changed" + +class TaskEventListener: + def handle(self, event: TaskEvent, task: "Task", payload: Optional[Any] = None): + pass + +class Task(ABC): + def __init__(self, name: str): + self.name = name + self.status = Status.PENDING + self.inputs: List[InPort] = [] + self.outputs: List[OutPort] = [] + self._listeners: List[TaskEventListener] = [] + + # Events + def add_listener(self, listener: TaskEventListener): + self._listeners.append(listener) + + def _emit(self, event: TaskEvent, payload: Optional[Any] = None): + for listener in self._listeners: + listener.handle(event, self, payload) + + def _set_status(self, new_status: Status): + old_status = self.status + self.status = new_status + if old_status != new_status: + self._emit(TaskEvent.STATUS_CHANGED, {"from": old_status, "to": new_status}) + + # Lifecycle + def pause(self): + if self.status == Status.RUNNING: + self._set_status(Status.PAUSED) + + def resume(self): + if self.status == Status.PAUSED: + self._set_status(Status.READY) + + def stop(self): + if self.status in (Status.RUNNING, Status.PAUSED, Status.READY): + self._set_status(Status.STOPPED) + + # Ports + def _add_input_port(self, port: InPort): + port.parent_task = self + self.inputs.append(port) + + def _add_output_port(self, port: OutPort): + port.parent_task = self + self.outputs.append(port) + + def check_inputs_ready(self, context: ExecutionContext) -> bool: + """Check if all required inputs have data in ExecutionContext.""" + for in_port in self.inputs: + if in_port.required and not context.has_data(in_port): + return False + return True + + # Runnable + @abstractmethod + def run(self, context: ExecutionContext): + pass diff --git a/Dagster/Radiuma_API/radiuma_api/workflow.py b/Dagster/Radiuma_API/radiuma_api/workflow.py new file mode 100644 index 0000000..1e3664b --- /dev/null +++ b/Dagster/Radiuma_API/radiuma_api/workflow.py @@ -0,0 +1,44 @@ +from typing import List +from radiuma_api.task import Task, Status, TaskEvent +from radiuma_api.execution_context import ExecutionContext +from radiuma_api.scheduler import Scheduler +from radiuma_api.io_port import CompatibilityException +# Write Codes Here: + + +class Workflow(Task): + """Composite node: aggregates Tasks (Modules or other Workflows).""" + + def __init__(self, name: str): + super().__init__(name) + self.children: List[Task] = [] + + def add_child(self, task: Task): + task.parent_task = self + self.children.append(task) + + def run(self, context: ExecutionContext): + """Delegate execution of children to Scheduler.""" + self._set_status(Status.RUNNING) + scheduler = Scheduler() + + for child in self.children: + if self.status == Status.STOPPED: + break + + try: + if child.check_inputs_ready(context): + scheduler.run(child, context) + else: + print(f"Task {child.name} not ready, skipping.") + except CompatibilityException as e: + self._emit(TaskEvent.ON_ERROR, {"error": e.reason, "task": child.name}) + child._set_status(Status.FAILED) + + # Final status evaluation + if all(c.status == Status.COMPLETED for c in self.children): + self._set_status(Status.COMPLETED) + elif any(c.status == Status.FAILED for c in self.children): + self._set_status(Status.FAILED) + else: + self._set_status(Status.STOPPED) diff --git a/Dagster/Radiuma_API/rnd/run_rnd_workflow.py b/Dagster/Radiuma_API/rnd/run_rnd_workflow.py new file mode 100644 index 0000000..b885822 --- /dev/null +++ b/Dagster/Radiuma_API/rnd/run_rnd_workflow.py @@ -0,0 +1,69 @@ +from radiuma_api.workflow import Workflow +from radiuma_api.execution_context import ExecutionContext +from radiuma_api.io_port import InPort, OutPort, ImageType, TextType +from radiuma_api.module import Module +from radiuma_api.task import Status +from radiuma_api.asset import Asset +# Codes Go Below: + + +class Reader(Module): + def __init__(self): + super().__init__("Reader") + self._add_output_port(OutPort("images", ImageType())) + + def run(self, context): + print("[Reader] Loading images...") + context.put(self.outputs[0], Asset(["img1", "img2"], "images")) + self._set_status(Status.COMPLETED) + + +class Extractor(Module): + def __init__(self): + super().__init__("Extractor") + self._add_input_port(InPort("images", ImageType())) + self._add_output_port(OutPort("features", TextType())) + + def run(self, context): + imgs = context.get(self.inputs[0]).data + print(f"[Extractor] Extracting features from: {imgs}") + feats = [{"img": img, "feat": f"feature_of_{img}"} for img in imgs] + context.put(self.outputs[0], Asset(feats, "features")) + self._set_status(Status.COMPLETED) + + +class Writer(Module): + def __init__(self): + super().__init__("Writer") + self._add_input_port(InPort("features", TextType())) + + def run(self, context): + feats = context.get(self.inputs[0]).data + print("[Writer] Saving features:") + for f in feats: + print(" ", f) + self._set_status(Status.COMPLETED) + + +def main(): + ctx = ExecutionContext() + wf = Workflow("RND_Workflow") + + r = Reader() + e = Extractor() + w = Writer() + + r.outputs[0].connect(e.inputs[0]) + e.outputs[0].connect(w.inputs[0]) + + wf.add_child(r) + wf.add_child(e) + wf.add_child(w) + + print("=== Running Workflow ===") + wf.run(ctx) + print("=== Workflow Completed ===") + + +if __name__ == "__main__": + main() diff --git a/Dagster/Radiuma_API/tests/__pycache__/test_composite_type.cpython-313-pytest-9.0.2.pyc b/Dagster/Radiuma_API/tests/__pycache__/test_composite_type.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..882b32b Binary files /dev/null and b/Dagster/Radiuma_API/tests/__pycache__/test_composite_type.cpython-313-pytest-9.0.2.pyc differ diff --git a/Dagster/Radiuma_API/tests/__pycache__/test_execution_context.cpython-313-pytest-9.0.2.pyc b/Dagster/Radiuma_API/tests/__pycache__/test_execution_context.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..bed0356 Binary files /dev/null and b/Dagster/Radiuma_API/tests/__pycache__/test_execution_context.cpython-313-pytest-9.0.2.pyc differ diff --git a/Dagster/Radiuma_API/tests/__pycache__/test_ports.cpython-313-pytest-9.0.2.pyc b/Dagster/Radiuma_API/tests/__pycache__/test_ports.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..f5c62a6 Binary files /dev/null and b/Dagster/Radiuma_API/tests/__pycache__/test_ports.cpython-313-pytest-9.0.2.pyc differ diff --git a/Dagster/Radiuma_API/tests/__pycache__/test_scheduler.cpython-313-pytest-9.0.2.pyc b/Dagster/Radiuma_API/tests/__pycache__/test_scheduler.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..7ceac1e Binary files /dev/null and b/Dagster/Radiuma_API/tests/__pycache__/test_scheduler.cpython-313-pytest-9.0.2.pyc differ diff --git a/Dagster/Radiuma_API/tests/__pycache__/test_task.cpython-313-pytest-9.0.2.pyc b/Dagster/Radiuma_API/tests/__pycache__/test_task.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..694c155 Binary files /dev/null and b/Dagster/Radiuma_API/tests/__pycache__/test_task.cpython-313-pytest-9.0.2.pyc differ diff --git a/Dagster/Radiuma_API/tests/__pycache__/test_workflow.cpython-313-pytest-9.0.2.pyc b/Dagster/Radiuma_API/tests/__pycache__/test_workflow.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..569b672 Binary files /dev/null and b/Dagster/Radiuma_API/tests/__pycache__/test_workflow.cpython-313-pytest-9.0.2.pyc differ diff --git a/Dagster/Radiuma_API/tests/test_composite_type.py b/Dagster/Radiuma_API/tests/test_composite_type.py new file mode 100644 index 0000000..18b401a --- /dev/null +++ b/Dagster/Radiuma_API/tests/test_composite_type.py @@ -0,0 +1,43 @@ +from radiuma_api.io_port import ( + CompositeType, + ImageType, + TextType, + TableType, + CompatibilityException +) + +def test_composite_type_success(): + t1 = CompositeType({ + "img": ImageType(), + "meta": TextType(language="en") + }) + + t2 = CompositeType({ + "img": ImageType(), + "meta": TextType(language="en") + }) + + # Must succeed + t1.can_connect_to(t2) + + +def test_composite_type_key_mismatch(): + t1 = CompositeType({"img": ImageType()}) + t2 = CompositeType({"image": ImageType()}) + + try: + t1.can_connect_to(t2) + assert False, "Expected CompositeType key mismatch" + except CompatibilityException: + assert True + + +def test_composite_type_internal_type_mismatch(): + t1 = CompositeType({"img": ImageType(modality="nifti")}) + t2 = CompositeType({"img": ImageType(modality="dicom")}) + + try: + t1.can_connect_to(t2) + assert False, "Expected internal DType mismatch" + except CompatibilityException: + assert True diff --git a/Dagster/Radiuma_API/tests/test_execution_context.py b/Dagster/Radiuma_API/tests/test_execution_context.py new file mode 100644 index 0000000..232410c --- /dev/null +++ b/Dagster/Radiuma_API/tests/test_execution_context.py @@ -0,0 +1,17 @@ +from radiuma_api.execution_context import ExecutionContext +from radiuma_api.io_port import OutPort, InPort, ImageType +from radiuma_api.asset import Asset +# Codes Go Below: + + +def test_context_put_get(): + ctx = ExecutionContext() + outp = OutPort("o", ImageType()) + inp = InPort("i", ImageType()) + + outp.connect(inp) + + asset = Asset(["img1"], "images") + ctx.put(outp, asset) + + assert ctx.get(inp).data == ["img1"] diff --git a/Dagster/Radiuma_API/tests/test_ports.py b/Dagster/Radiuma_API/tests/test_ports.py new file mode 100644 index 0000000..cb7f322 --- /dev/null +++ b/Dagster/Radiuma_API/tests/test_ports.py @@ -0,0 +1,39 @@ +from radiuma_api.io_port import InPort, OutPort, ImageType, TextType, CompatibilityException +# Codes Go Below: + + +def test_compatible_ports(): + """Ports with the same DType must connect successfully.""" + outp = OutPort("o", ImageType()) + inp = InPort("i", ImageType()) + + outp.connect(inp) + + assert inp.connected_output == outp + + +def test_incompatible_ports(): + """Connecting mismatched DTypes must raise CompatibilityException.""" + outp = OutPort("o", ImageType()) + inp = InPort("i", TextType()) + + try: + outp.connect(inp) + assert False, "Expected CompatibilityException" + except CompatibilityException: + assert True + + +def test_outport_single_connection_limit(): + """OutPort must not allow more than one connection.""" + outp = OutPort("o", ImageType()) + inp1 = InPort("i1", ImageType()) + inp2 = InPort("i2", ImageType()) + + outp.connect(inp1) + + try: + outp.connect(inp2) + assert False, "Expected OutPort max connections error" + except CompatibilityException: + assert True diff --git a/Dagster/Radiuma_API/tests/test_scheduler.py b/Dagster/Radiuma_API/tests/test_scheduler.py new file mode 100644 index 0000000..e13fcbf --- /dev/null +++ b/Dagster/Radiuma_API/tests/test_scheduler.py @@ -0,0 +1,43 @@ +from radiuma_api.scheduler import Scheduler +from radiuma_api.workflow import Workflow +from radiuma_api.module import Module +from radiuma_api.io_port import InPort, OutPort, ImageType +from radiuma_api.execution_context import ExecutionContext +from radiuma_api.task import Status +# Codes Go below: + + +class Reader(Module): + def __init__(self): + super().__init__("Reader") + self._add_output_port(OutPort("images", ImageType())) + + def run(self, context): + context.put(self.outputs[0], "DATA") + self._set_status(Status.COMPLETED) + +class Extractor(Module): + def __init__(self): + super().__init__("Extractor") + self._add_input_port(InPort("images", ImageType())) + + def run(self, context): + assert context.get(self.inputs[0]) == "DATA" + self._set_status(Status.COMPLETED) + +def test_scheduler_runs_in_order(): + ctx = ExecutionContext() + wf = Workflow("WF") + + r = Reader() + e = Extractor() + + r.outputs[0].connect(e.inputs[0]) + + wf.add_child(r) + wf.add_child(e) + + wf.run(ctx) + + assert r.status == Status.COMPLETED + assert e.status == Status.COMPLETED diff --git a/Dagster/Radiuma_API/tests/test_task.py b/Dagster/Radiuma_API/tests/test_task.py new file mode 100644 index 0000000..e6b1195 --- /dev/null +++ b/Dagster/Radiuma_API/tests/test_task.py @@ -0,0 +1,24 @@ +from radiuma_api.task import Status +from radiuma_api.module import Module +from radiuma_api.execution_context import ExecutionContext +from radiuma_api.io_port import OutPort, ImageType +# Codes Go below: + + +class Dummy(Module): + def __init__(self): + super().__init__("Dummy") + self._add_output_port(OutPort("o", ImageType())) + + def run(self, context): + self._set_status(Status.RUNNING) + context.put(self.outputs[0], None) + self._set_status(Status.COMPLETED) + +def test_task_lifecycle(): + ctx = ExecutionContext() + t = Dummy() + + assert t.status == Status.PENDING + t.run(ctx) + assert t.status == Status.COMPLETED diff --git a/Dagster/Radiuma_API/tests/test_workflow.py b/Dagster/Radiuma_API/tests/test_workflow.py new file mode 100644 index 0000000..cde8422 --- /dev/null +++ b/Dagster/Radiuma_API/tests/test_workflow.py @@ -0,0 +1,57 @@ +from radiuma_api.workflow import Workflow +from radiuma_api.execution_context import ExecutionContext +from radiuma_api.io_port import InPort, OutPort, ImageType, TextType +from radiuma_api.module import Module +from radiuma_api.task import Status +from radiuma_api.asset import Asset +# Codes Go below: + + +class Reader(Module): + def __init__(self): + super().__init__("Reader") + self._add_output_port(OutPort("images", ImageType())) + + def run(self, context): + context.put(self.outputs[0], Asset(["img1"], "images")) + self._set_status(Status.COMPLETED) + +class Extractor(Module): + def __init__(self): + super().__init__("Extractor") + self._add_input_port(InPort("images", ImageType())) + self._add_output_port(OutPort("features", TextType())) + + def run(self, context): + imgs = context.get(self.inputs[0]).data + context.put(self.outputs[0], Asset([{"img": imgs[0]}], "features")) + self._set_status(Status.COMPLETED) + +class Writer(Module): + def __init__(self): + super().__init__("Writer") + self._add_input_port(InPort("features", TextType())) + + def run(self, context): + feats = context.get(self.inputs[0]).data + assert feats[0]["img"] == "img1" + self._set_status(Status.COMPLETED) + +def test_end_to_end(): + ctx = ExecutionContext() + wf = Workflow("WF") + + r = Reader() + e = Extractor() + w = Writer() + + r.outputs[0].connect(e.inputs[0]) + e.outputs[0].connect(w.inputs[0]) + + wf.add_child(r) + wf.add_child(e) + wf.add_child(w) + + wf.run(ctx) + + assert w.status == Status.COMPLETED diff --git a/Galaxy/README.md b/Galaxy/README.md new file mode 100644 index 0000000..e508286 --- /dev/null +++ b/Galaxy/README.md @@ -0,0 +1,17 @@ +# Galaxy Project + +This folder contains prototypes and files related to the **Galaxy Workflow Engine**. + +The goal of this section is to define and run simple workflows to demonstrate Galaxy's capabilities in managing and reproducing scientific processes. + +## Quick Start +To run a sample workflow: +1. Prepare a workflow file (e.g. `workflow.xml`). +2. Run Galaxy and load the workflow. + +## Folder Structure +- `workflow.xml`: A prototype of a simple workflow +- Other tools and supporting files will be added in the future + +## Description +Galaxy is a powerful platform for performing scientific and data analytics. In this project, we use it to create reproducible and auditable workflows. \ No newline at end of file diff --git a/Galaxy/docker-compose.yml b/Galaxy/docker-compose.yml new file mode 100644 index 0000000..17236db --- /dev/null +++ b/Galaxy/docker-compose.yml @@ -0,0 +1,48 @@ +version: "3.9" + +services: + galaxy: + image: quay.io/bgruening/galaxy:24.2 + container_name: galaxy_radiuma + ports: + - "8080:80" # رابط وب Galaxy + environment: + - GALAXY_CONFIG_BRAND=Radiuma Galaxy + - GALAXY_CONFIG_ADMIN_USERS=admin@radiuma.org + - GALAXY_CONFIG_REQUIREMENTS_FILE=/galaxy/config/requirements.txt + - GALAXY_CONFIG_BOOTSTRAP_ADMIN_API_KEY=radiuma_admin_key # ✅ جایگزین master_api_key + - GALAXY_CONFIG_ENABLE_FTP_UPLOAD_DIR=True + - GALAXY_CONFIG_FTP_UPLOAD_DIR=/ftp_uploads # ✅ مسیر داخلی FTP + - GALAXY_CONFIG_TOOL_CONFIG_FILE=/galaxy/config/tool_conf.xml + - GALAXY_CONFIG_LOG_LEVEL=DEBUG + - GALAXY_CONFIG_WATCH_TOOLS=False + - GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:galaxy@postgres:5432/galaxy + volumes: + - galaxy_data:/galaxy/database + - ./galaxy/tools:/galaxy/tools + - ./requirements.txt:/galaxy/config/requirements.txt + - ./galaxy/config/tool_conf.xml:/galaxy/config/tool_conf.xml + - ./ftp:/ftp_uploads # ✅ mount صحیح برای FTP + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + + postgres: + image: postgres:15 + container_name: galaxy_postgres + environment: + POSTGRES_USER: galaxy + POSTGRES_PASSWORD: galaxy + POSTGRES_DB: galaxy + volumes: + - ./galaxy_db:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD", "pg_isready", "-U", "galaxy", "-h", "postgres"] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + galaxy_data: \ No newline at end of file diff --git a/Galaxy/galaxy/config/tool_conf.xml b/Galaxy/galaxy/config/tool_conf.xml new file mode 100644 index 0000000..e480a0c --- /dev/null +++ b/Galaxy/galaxy/config/tool_conf.xml @@ -0,0 +1,5 @@ +
+ + + +
diff --git a/Galaxy/galaxy/tools/pysera_tool.xml b/Galaxy/galaxy/tools/pysera_tool.xml new file mode 100644 index 0000000..4fa08e6 --- /dev/null +++ b/Galaxy/galaxy/tools/pysera_tool.xml @@ -0,0 +1,24 @@ + + Standardized radiomics feature extraction using PySERA + + + + + + + + + + + + + + diff --git a/Galaxy/galaxy/tools/radiuma/classification.xml b/Galaxy/galaxy/tools/radiuma/classification.xml new file mode 100644 index 0000000..b10bfd8 --- /dev/null +++ b/Galaxy/galaxy/tools/radiuma/classification.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/Galaxy/galaxy/tools/radiuma/classifiers.py b/Galaxy/galaxy/tools/radiuma/classifiers.py new file mode 100644 index 0000000..6731c27 --- /dev/null +++ b/Galaxy/galaxy/tools/radiuma/classifiers.py @@ -0,0 +1,1038 @@ +from pandas.core.common import random_state +from sklearn.kernel_ridge import KernelRidge +from sklearn.neural_network import MLPClassifier +from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor +from sklearn.svm import SVC, SVR +from sklearn.linear_model import LogisticRegression, BayesianRidge, LinearRegression, Lasso, Ridge, ElasticNet +from sklearn.gaussian_process import GaussianProcessClassifier, GaussianProcessRegressor +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, BaggingClassifier, ExtraTreesRegressor, \ + GradientBoostingRegressor +from sklearn.naive_bayes import GaussianNB +from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis +from sklearn.metrics import confusion_matrix +from sklearn.metrics import accuracy_score +from sklearn.metrics import recall_score +from sklearn.metrics import precision_score +from sklearn.metrics import f1_score +from sklearn.metrics import roc_auc_score +from sklearn.metrics import auc +from sklearn.metrics import roc_curve +import numpy as np +import logging +from skopt import BayesSearchCV +from skopt.space import Real, Categorical, Integer + + +class Classifiers: + + def __init__(self, X_train=None, y_train=None, X_val=None, y_val=None, X_test=None, y_test=None): + # self.AlgName = AlgName + # self.config = config + self.X_train = X_train + self.y_train = y_train + self.X_val = X_val + self.y_val = y_val + self.X_test = X_test + self.y_test = y_test + self.model = None + self.regressors = { + "BayesianRidge": BayesianRidge, + "GaussianProcessRegressor": GaussianProcessRegressor, + "KernelRidge": lambda: KernelRidge(kernel="rbf"), + "KNeighborsRegressor": KNeighborsRegressor, + "DecisionTreeRegressor": DecisionTreeRegressor, + "ExtraTreesRegressor": ExtraTreesRegressor, + "LinearRegression": LinearRegression, + "Lasso": Lasso, + "Ridge": Ridge, + "ElasticNet": ElasticNet, + "SVR": SVR, + "GradientBoostingRegressor": GradientBoostingRegressor, + } + + def Vs_fit_and_predict_model(self): + try: + # Validate input dimensions + if self.X_train is None or self.X_train.shape[0] == 0 or self.X_train.shape[1] == 0: + print(f"Error: Invalid X_train dimensions: {self.X_train.shape if self.X_train is not None else 'None'}") + return None, None, None + + # Convert y_train to 1D array if it's not None + if self.y_train is not None: + if hasattr(self.y_train, 'values'): + y_train_1d = self.y_train.values.ravel() + elif hasattr(self.y_train, 'ravel'): + y_train_1d = self.y_train.ravel() + else: + y_train_1d = np.array(self.y_train).ravel() + + # Validate dimensions match + if len(y_train_1d) != self.X_train.shape[0]: + print(f"Error: Dimension mismatch - X_train: {self.X_train.shape[0]} samples, y_train: {len(y_train_1d)} labels") + return None, None, None + + self.model.fit(self.X_train, y_train_1d) + else: + # Handle case when y_train is None + return None, None, None + + # Validate X_val dimensions before prediction + if self.X_val is not None and self.X_val.shape[1] != self.X_train.shape[1]: + print(f"Error: Feature count mismatch - X_train: {self.X_train.shape[1]}, X_val: {self.X_val.shape[1]}") + return None, None, None + + y_train_pred = self.model.predict(self.X_train) + y_val_pred = self.model.predict(self.X_val) if self.X_val is not None else None + + if self.X_test is not None: + # Validate X_test dimensions before prediction + if self.X_test.shape[1] != self.X_train.shape[1]: + print(f"Error: Feature count mismatch - X_train: {self.X_train.shape[1]}, X_test: {self.X_test.shape[1]}") + y_test_pred = None + else: + y_test_pred = self.model.predict(self.X_test) + else: + y_test_pred = None + + return y_train_pred, y_val_pred, y_test_pred + + except Exception as e: + print(f"Error in model fitting/prediction: {e}") + return None, None, None + + def _safe_extract_cm_elements(self, cm): + """ + Safely extract tn, fp, fn, tp values from confusion matrix of any size. + Handles 1x1, 2x2, and larger confusion matrices appropriately. + + Args: + cm: confusion matrix from sklearn.metrics.confusion_matrix + + Returns: + tuple: (tn, fp, fn, tp) with appropriate defaults for missing elements + """ + try: + # Get matrix dimensions + rows, cols = cm.shape + + # Log unusual matrix dimensions for debugging + if rows == 1 and cols == 1: + logging.warning(f"Single-class prediction detected. Confusion matrix shape: {cm.shape}, matrix: {cm}") + # All predictions are the same class + # tn = count of correct predictions, others are 0 + return cm[0, 0], 0, 0, 0 + elif rows != 2 or cols != 2: + logging.info(f"Non-standard confusion matrix dimensions: {cm.shape}") + + # For 2x2 matrix (standard binary classification) + if rows >= 2 and cols >= 2: + # Extract standard binary classification metrics + tn = cm[0, 0] if rows > 0 and cols > 0 else 0 + fp = cm[0, 1] if rows > 0 and cols > 1 else 0 + fn = cm[1, 0] if rows > 1 and cols > 0 else 0 + tp = cm[1, 1] if rows > 1 and cols > 1 else 0 + return tn, fp, fn, tp + + # For other edge cases, return zeros + else: + logging.warning(f"Unusual confusion matrix shape {cm.shape}, returning zero values") + return 0, 0, 0, 0 + + except Exception as e: + # Log the error and return safe defaults + logging.error(f"Error extracting confusion matrix elements: {e}") + return 0, 0, 0, 0 + + def Vs_Score_cls(self, y, y_pred): + """ + Calculate classification scores with robust confusion matrix handling. + + Args: + y: true labels + y_pred: predicted labels + + Returns: + dict: classification metrics including tn, fp, fn, tp, accuracy, etc. + """ + try: + # Input validation + if len(y) == 0 or len(y_pred) == 0: + logging.warning("Empty input arrays for classification scoring") + return { + 'tn': 0, 'fp': 0, 'fn': 0, 'tp': 0, + 'acc': 0.0, 'MisclassificationRate': 1.0, 're': 0.0, + 'Sensitivity': 0.0, 'pre': 0.0, 'f_sc': 0.0, 'AUC': 0.0, 'Specificity': 0.0 + } + + # Check for single-class scenarios + unique_y = np.unique(y) + unique_y_pred = np.unique(y_pred) + + if len(unique_y_pred) == 1: + logging.warning(f"Single-class prediction detected: all predictions are class {unique_y_pred[0]}") + + cm = confusion_matrix(y, y_pred) + + ac = accuracy_score(y, y_pred) + MisclassificationRate = 1 - ac + + # Safely extract confusion matrix elements + tn, fp, fn, tp = self._safe_extract_cm_elements(cm) + + # Calculate metrics with zero division handling + re = recall_score(y, y_pred, average='weighted', zero_division=0) + Sensitivity = re + + pr = precision_score(y, y_pred, average='weighted', zero_division=0) + fs = f1_score(y, y_pred, average='weighted', zero_division=0) + + # Calculate Specificity + if tn + fp > 0: + Specificity = tn / (tn + fp) + else: + Specificity = 0.0 + + # Calculate AUC (for binary classification) + try: + from sklearn.metrics import roc_auc_score + if len(unique_y) == 2 and len(unique_y_pred) == 2: + auc = roc_auc_score(y, y_pred) + else: + auc = 0.0 + except Exception as e: + logging.warning(f"Could not calculate AUC: {e}") + auc = 0.0 + + # Log successful completion for debugging + logging.debug(f"Classification metrics calculated successfully. Accuracy: {ac:.3f}") + + return { + 'tn': tn, 'fp': fp, 'fn': fn, 'tp': tp, + 'acc': ac, 'MisclassificationRate': MisclassificationRate, 're': re, + 'Sensitivity': Sensitivity, 'pre': pr, 'f_sc': fs, 'AUC': auc, 'Specificity': Specificity + } + + except Exception as e: + logging.error(f"Error in Vs_Score_cls: {e}", exc_info=True) + # Return safe defaults to prevent crashes + return { + 'tn': 0, 'fp': 0, 'fn': 0, 'tp': 0, + 'acc': 0.0, 'MisclassificationRate': 1.0, 're': 0.0, + 'Sensitivity': 0.0, 'pre': 0.0, 'f_sc': 0.0, 'AUC': 0.0, 'Specificity': 0.0 + } + + def Vs_Score_cls_All_Data(self, y_train_pred, y_val_pred, y_test_pred): + + # Check if y_train is already a numpy array or has to_numpy method + y_train_np = self.y_train if isinstance(self.y_train, np.ndarray) else (self.y_train.to_numpy() if hasattr(self.y_train, 'to_numpy') else np.array(self.y_train)) + train_score_dict = self.Vs_Score_cls(y_train_np, y_train_pred) + + # Check if y_val is already a numpy array or has to_numpy method + y_val_np = self.y_val if isinstance(self.y_val, np.ndarray) else (self.y_val.to_numpy() if hasattr(self.y_val, 'to_numpy') else np.array(self.y_val)) + val_score_dict = self.Vs_Score_cls(y_val_np, y_val_pred) + + if y_test_pred is not None and self.y_test is not None: + # Check if y_test is already a numpy array or has to_numpy method + y_test_np = self.y_test if isinstance(self.y_test, np.ndarray) else (self.y_test.to_numpy() if hasattr(self.y_test, 'to_numpy') else np.array(self.y_test)) + test_score_dict = self.Vs_Score_cls(y_test_np, y_test_pred) + else: + test_score_dict = None + + return train_score_dict, val_score_dict, test_score_dict + + # def SelectAlg(self): + # import json + # f = open('data.json') + # JSONdata = json.load(f) + # f.close() + # algn = self.AlgName + # if self.AlgName == "LogisticRegression": + + # self.Vs_LogisticClassifier( + # JSONdata[algn][0], + # JSONdata[algn][1], + # JSONdata[algn][2], + # JSONdata[algn][3], + # JSONdata[algn][4], + # JSONdata[algn][5] + # ) + # elif self.AlgName == "BaggingClassifier": + # self.Vs_BaggingClassifier( + # JSONdata[algn][0], + # JSONdata[algn][1], + # JSONdata[algn][2], + # JSONdata[algn][3], + # JSONdata[algn][4] + # ) + # elif self.AlgName == "AdaBoostClassifier": + # self.Vs_AdaBoostClassifier( + # JSONdata[algn][0], + # JSONdata[algn][1], + # JSONdata[algn][2], + # JSONdata[algn][3], + # JSONdata[algn][4] + # ) + # elif self.AlgName == "KNeighborsClassifier": + # self.Vs_KNeighborsClassifier( + # JSONdata[algn][0], + # JSONdata[algn][1], + # JSONdata[algn][2], + # JSONdata[algn][3], + # JSONdata[algn][4], + # JSONdata[algn][5] + # ) + # else: + # print("Algorithm name is wrong") + + def Vs_LogisticClassifier(self, + penalty="l2", + C=1.0, + class_weight=None, + solver="lbfgs", + max_iter=100, + multi_class="auto", + random_state=None + ): + + solver = solver.lower() + multi_class = multi_class.lower() + + if penalty == "None" or penalty is None: + penalty = None + else: + penalty = penalty.lower() + + if class_weight == "None" or class_weight is None: + class_weight = None + else: + class_weight = class_weight.lower() + + model = LogisticRegression( + penalty =penalty, + C=C, + class_weight=class_weight, + solver=solver, + max_iter=max_iter, + multi_class=multi_class, + random_state=random_state + ) + self.model = model + y_train_pred, y_val_pred, y_test_pred = self.Vs_fit_and_predict_model() + + train_score_dict, val_score_dict, test_score_dict = self.Vs_Score_cls_All_Data(y_train_pred, y_val_pred, + y_test_pred) + + return {'x_train': self.X_train, 'x_val': self.X_val, 'x_test': self.X_test, + 'y_train_pred': y_train_pred, 'y_val_pred': y_val_pred, 'y_test_pred': y_test_pred, + 'y_train': np.array(self.y_train), 'y_val': np.array(self.y_val), 'y_test': np.array(self.y_test), + 'train_score_dict': train_score_dict, 'val_score_dict': val_score_dict, + 'test_score_dict': test_score_dict, 'model_params': model.get_params()} + + def Vs_BaggingClassifier(self, + estimator=None, + n_estimators=10, + max_samples=1.0, + max_features=1.0, + bootstrap=True, + random_state=None): + + estimator_mapping = { + 'SVC': SVC, + 'DecisionTreeClassifier': DecisionTreeClassifier, + 'KNeighborsClassifier': KNeighborsClassifier, + 'LogisticRegression': LogisticRegression, + 'None': None + } + + estimator_class = estimator_mapping.get(estimator) + + # Create an instance of the estimator if it's not None + estimator_instance = None + if estimator_class is not None: + estimator_instance = estimator_class() + + if bootstrap == "True": + bootstrap = True + else: + bootstrap = False + + model = BaggingClassifier( + estimator=estimator_instance, + n_estimators=n_estimators, + max_samples=max_samples, + max_features=max_features, + bootstrap=bootstrap, + random_state=random_state + ) + self.model = model + y_train_pred, y_val_pred, y_test_pred = self.Vs_fit_and_predict_model() + + train_score_dict, val_score_dict, test_score_dict = self.Vs_Score_cls_All_Data(y_train_pred, y_val_pred, + y_test_pred) + + return {'x_train': self.X_train, 'x_val': self.X_val, 'x_test': self.X_test, + 'y_train_pred': y_train_pred, 'y_val_pred': y_val_pred, 'y_test_pred': y_test_pred, + 'y_train': np.array(self.y_train), 'y_val': np.array(self.y_val), 'y_test': np.array(self.y_test), + 'train_score_dict': train_score_dict, 'val_score_dict': val_score_dict, + 'test_score_dict': test_score_dict, 'model_params': model.get_params()} + + def Vs_AdaBoostClassifier(self, + estimator=None, + n_estimators=50, + learning_rate=1.0, + algorithm="SAMME.R", + random_state=None): + + estimator_mapping = { + 'SVC': SVC, + 'DecisionTreeClassifier': DecisionTreeClassifier, + 'KNeighborsClassifier': KNeighborsClassifier, + 'LogisticRegression': LogisticRegression, + 'None': None + } + + estimator_class = estimator_mapping.get(estimator) + + # Create an instance of the estimator if it's not None + estimator_instance = None + if estimator_class is not None: + estimator_instance = estimator_class() + + model = AdaBoostClassifier( + estimator=estimator_instance, + n_estimators=n_estimators, + learning_rate=learning_rate, + algorithm=algorithm, + random_state=random_state + ) + self.model = model + y_train_pred, y_val_pred, y_test_pred = self.Vs_fit_and_predict_model() + train_score_dict, val_score_dict, test_score_dict = self.Vs_Score_cls_All_Data(y_train_pred, y_val_pred, + y_test_pred) + + return {'x_train': self.X_train, 'x_val': self.X_val, 'x_test': self.X_test, + 'y_train_pred': y_train_pred, 'y_val_pred': y_val_pred, 'y_test_pred': y_test_pred, + 'y_train': np.array(self.y_train), 'y_val': np.array(self.y_val), 'y_test': np.array(self.y_test), + 'train_score_dict': train_score_dict, 'val_score_dict': val_score_dict, + 'test_score_dict': test_score_dict, 'model_params': model.get_params()} + + def Vs_KNeighborsClassifier(self, + n_neighbors=5, + weights="uniform", + algorithm="auto", + p=2, + metric="minkowski"): + + if weights is not None: + weights = weights.lower() + + algorithm = algorithm.lower() + metric = metric.lower() + + model = KNeighborsClassifier(n_neighbors=n_neighbors, + weights=weights, + algorithm=algorithm, + p=p, + metric=metric) + self.model = model + y_train_pred, y_val_pred, y_test_pred = self.Vs_fit_and_predict_model() + + train_score_dict, val_score_dict, test_score_dict = self.Vs_Score_cls_All_Data(y_train_pred, y_val_pred, + y_test_pred) + + return {'x_train': self.X_train, 'x_val': self.X_val, 'x_test': self.X_test, + 'y_train_pred': y_train_pred, 'y_val_pred': y_val_pred, 'y_test_pred': y_test_pred, + 'y_train': np.array(self.y_train), 'y_val': np.array(self.y_val), 'y_test': np.array(self.y_test), + 'train_score_dict': train_score_dict, 'val_score_dict': val_score_dict, + 'test_score_dict': test_score_dict, 'model_params': model.get_params()} + + def Vs_LogisticClassifier_pipeline(self, + penalty="l2", + C=1.0, + class_weight=None, + solver="lbfgs", + max_iter=100, + multi_class="auto", + random_state=None + ): + + solver = solver.lower() + multi_class = multi_class.lower() + + if penalty == "None" or penalty is None: + penalty = None + else: + penalty = penalty.lower() + + if class_weight == "None" or class_weight is None: + class_weight = None + else: + class_weight = class_weight.lower() + + model = LogisticRegression( + penalty =penalty, + C=C, + class_weight=class_weight, + solver=solver, + max_iter=max_iter, + multi_class=multi_class, + random_state=random_state + ) + + params = { + # 'model__penalty': ['l1', 'l2', 'elasticnet', 'none'], + 'model__C': [0.1, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0], + 'model__solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'] + } + + paramsAda = { + 'model__C': (0.0000001, 100.0), + 'model__solver': Categorical(['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']), + # 'model__penalty': Categorical(['l1', 'l2', 'elasticnet', 'none']), + + } + + model_type = 'logistic_regression' + return [model, paramsAda, params, model_type] + + def Vs_BaggingClassifier_pipeline(self, + estimator=None, + n_estimators=10, + max_samples=1.0, + max_features=1.0, + bootstrap=True, + random_state=None): + estimator_mapping = { + 'SVC': SVC, + 'DecisionTreeClassifier': DecisionTreeClassifier, + 'KNeighborsClassifier': KNeighborsClassifier, + 'LogisticRegression': LogisticRegression, + 'None': None + } + + estimator_class = estimator_mapping.get(estimator) + + # Create an instance of the estimator if it's not None + estimator_instance = None + if estimator_class is not None: + estimator_instance = estimator_class() + + if bootstrap == "True": + bootstrap = True + else: + bootstrap = False + + model = BaggingClassifier( + estimator=estimator_instance, + n_estimators=n_estimators, + max_samples=max_samples, + max_features=max_features, + bootstrap=bootstrap, + random_state=random_state + ) + + params = { + 'model__base_estimator': [SVC(), DecisionTreeClassifier()], + 'model__n_estimators': [3, 5, 10, 20, 50, 100] + } + + paramsAda = { + 'model__base_estimator': Categorical([SVC(kernel='rbf'), DecisionTreeClassifier()]), + 'model__n_estimators': (2, 250), + } + + model_type = 'bagging_classifier' + return [model, paramsAda, params, model_type] + + def Vs_AdaBoostClassifier_pipeline(self, + estimator=None, + n_estimators=50, + learning_rate=1.0, + algorithm="SAMME.R", + base_estimator=None, + random_state=None): + + estimator_mapping = { + 'SVC': SVC, + 'DecisionTreeClassifier': DecisionTreeClassifier, + 'KNeighborsClassifier': KNeighborsClassifier, + 'LogisticRegression': LogisticRegression, + 'None': None + } + + estimator_class = estimator_mapping.get(estimator) + + # Create an instance of the estimator if it's not None + estimator_instance = None + if estimator_class is not None: + estimator_instance = estimator_class() + + if base_estimator == "None" or base_estimator is None: + base_estimator = None + + model = AdaBoostClassifier( + estimator=estimator_instance, + n_estimators=n_estimators, + learning_rate=learning_rate, + algorithm=algorithm, + base_estimator=base_estimator, + random_state=random_state + ) + + base_models = [ + SVC(kernel='rbf'), + DecisionTreeClassifier()] + params = { + 'model__base_estimator': base_models, + 'model__n_estimators': [10, 20, 50, 100, 200, 400], + 'model__learning_rate': range(0.0001, 10.0, 0.1), + 'model__loss': ['linear', 'square', 'exponential'] + } + + paramsAda = { + 'model__base_estimator': Categorical([SVC(kernel='rbf'), DecisionTreeClassifier()]), + 'model__n_estimators': (2, 250), + 'model__learning_rate': (0.0001, 10.0), + 'model__loss': Categorical(['linear', 'square', 'exponential']), + } + + model_type = 'adaboost_classifier' + return [model, paramsAda, params, model_type] + + def Vs_KNeighborsClassifier_pipeline(self, + n_neighbors=5, + weights="uniform", + algorithm="auto", + leaf_size=30, + p=2, + metric="minkowski"): + + if weights is not None: + weights = weights.lower() + + algorithm = algorithm.lower() + metric = metric.lower() + + model = KNeighborsClassifier(n_neighbors=n_neighbors, + weights=weights, + algorithm=algorithm, + leaf_size=leaf_size, + p=p, + metric=metric + ) + + params = { + 'model__n_neighbors': range(1, 51), + 'model__leaf_size': range(5, 65, 5), + "model__metric": ["euclidean", "manhattan", "cityblock", "minkowski"], + 'model__algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'] + } + + paramsAda = { + 'model__n_neighbors': (2, 51), + 'model__leaf_size': (5, 65), + 'model__metric': Categorical(["euclidean", "manhattan", "cityblock", "minkowski"]), + 'model__algorithm': Categorical(['auto', 'ball_tree', 'kd_tree', 'brute']), + } + + model_type = 'k_neighbors_classifier' + return [model, paramsAda, params, model_type] + + def Vs_svmClassifier_Pipeline(self, + C=1.0, + kernel='rbf', + degree=3, + coef0=0.0, + class_wight=None, + gamma="scale", + decision_function_shape="ovr"): + if class_wight == "None" or class_wight is None: + class_wight = None + model = SVC( + C=C, + kernel=kernel, + degree=degree, + coef0=coef0, + class_weight=class_wight, + gamma=gamma, + decision_function_shape=decision_function_shape + ) + params = { + 'model__c': [0.1, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0], + 'model__kernel': ['linear', 'poly', 'rbf', 'sigmoid', 'precomputed'], + 'model__degree': (1, 180), + 'model__ceofo': (0.0, 1.0), + 'model__cache_size': (100.0, 300.0), + 'model__class_weight': [[1, 12, 41, 42], 'balanced', None], + 'model__verbos': [True, False], + 'model__max_iter': (1, 50) + } + paramsAda = { + 'model__c': [0.1, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0], + 'model__kernel': Categorical(['linear', 'poly', 'rbf', 'sigmoid', 'precomputed']), + 'model__degree': range(1, 180), + 'model__ceofo': range(0.0, 1.0), + 'model__cache_size': range(100.0, 300.0), + 'model__class_weight': Categorical([[1, 12, 41, 42], 'balanced', None]), + 'model__verbos': Categorical([True, False]), + 'model__max_iter': range(1, 50, 3) + } + model_type = 'svm_classifier' + return [model, model_type, params, paramsAda] + + def Vs_decision_tree_Pipline(self, + criterion='gini', + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + class_weight=None, + random_state=None): + + if max_depth == "None" or max_depth is None or max_depth == 0: + max_depth = None + if class_weight == "None" or class_weight is None: + class_weight = None + + model = DecisionTreeClassifier( + criterion=criterion, + max_depth=max_depth, + min_samples_split=min_samples_split, + min_samples_leaf=min_samples_leaf, + class_weight=class_weight, + random_state=random_state + ) + params = { + 'model__min_sample_spllit': range(1, 42, 2), + 'model__min_sample_leaf': range(0, 10) + } + paramsAda = { + 'model__min_sample_spllit': (1, 42), + 'model__min_sample_leaf': (0, 10) + } + model_type = 'decision_tree_classifier' + return [model, model_type, params, paramsAda] + + def VS_naive_bayes_Pipeline(self): + model = GaussianNB() + + return model + + def Vs_LogisticClassifier_Get_best_params(self, + penalty="l2", + C=1.0, + class_weight=None, + solver="lbfgs", + max_iter=100, + multi_class="auto", + best_parameters=None, + random_state=None + ): + + solver = solver.lower() + multi_class = multi_class.lower() + + if penalty == "None" or penalty is None: + penalty = None + else: + penalty = penalty.lower() + + if class_weight == "None" or class_weight is None: + class_weight = None + else: + class_weight = class_weight.lower() + + best_c = best_parameters['model__C'] + best_solver = best_parameters['model__solver'] + + model = LogisticRegression( + penalty=penalty, + C=best_c, + class_weight=class_weight, + solver=best_solver, + max_iter=max_iter, + multi_class=multi_class, + random_state=random_state + ) + + return model + + def Vs_BaggingClassifier_Get_best_params(self, + estimator=None, + n_estimators=10, + max_samples=1.0, + max_features=1.0, + bootstrap=True, + best_parameters=None, + random_state=None): + + # if estimator is not None: + # estimator = estimator.lower() + + estimator_mapping = { + 'SVC': SVC, + 'DecisionTreeClassifier': DecisionTreeClassifier, + 'KNeighborsClassifier': KNeighborsClassifier, + 'LogisticRegression': LogisticRegression, + 'None': None + } + + estimator_class = estimator_mapping.get(estimator) + + # Create an instance of the estimator if it's not None + estimator_instance = None + if estimator_class is not None: + estimator_instance = estimator_class() + + if bootstrap == "True": + bootstrap = True + else: + bootstrap = False + + best_base_estimator = best_parameters['model__base_estimator'] + best_n_estimators = best_parameters['model__n_estimators'] + + model = BaggingClassifier( + estimator=estimator_instance, + n_estimators=best_n_estimators, + base_estimator=best_base_estimator, + max_samples=max_samples, + max_features=max_features, + bootstrap=bootstrap, + random_state=random_state + ) + + return model + + def Vs_AdaBoostClassifier_Get_best_params(self, + estimator=None, + n_estimators=50, + learning_rate=1.0, + algorithm="SAMME.R", + base_estimator=None, + best_parameters=None, + random_state=None): + + estimator_mapping = { + 'SVC': SVC, + 'DecisionTreeClassifier': DecisionTreeClassifier, + 'KNeighborsClassifier': KNeighborsClassifier, + 'LogisticRegression': LogisticRegression, + 'None': None + } + + estimator_class = estimator_mapping.get(estimator) + + # Create an instance of the estimator if it's not None + estimator_instance = None + if estimator_class is not None: + estimator_instance = estimator_class() + + if base_estimator == "None" or base_estimator is None: + base_estimator = None + + best_base_estimator = best_parameters.get('model__base_estimator', base_estimator) + best_n_estimators = best_parameters.get('model__n_estimators', n_estimators) + best_learning_rate = best_parameters.get('model__learning_rate', learning_rate) + # Use algorithm parameter if model__loss is not in best_parameters + best_loss = best_parameters.get('model__loss', algorithm) + + model = AdaBoostClassifier( + estimator=estimator_instance, + base_estimator=best_base_estimator, + n_estimators=best_n_estimators, + learning_rate=best_learning_rate, + algorithm=algorithm, + random_state=random_state + ) + + return model + + def Vs_KNeighborsClassifier_Get_best_params(self, + n_neighbors=5, + weights="uniform", + algorithm="auto", + leaf_size=30, + p=2, + metric="minkowski", + best_parameters=None): + + if weights is not None: + weights = weights.lower() + + algorithm = algorithm.lower() + metric = metric.lower() + + best_n_neighbors = best_parameters['model__n_neighbors'] + best_leaf_size = best_parameters['model__leaf_size'] + best_metric = best_parameters['model__metric'] + best_algorithm = best_parameters['model__algorithm'] + + model = KNeighborsClassifier(n_neighbors=best_n_neighbors, + weights=weights, + algorithm=best_algorithm, + leaf_size=best_leaf_size, + p=p, + metric=best_metric + ) + + return model + + def Vs_decision_tree_Get_best_params(self, + criterion='gini', + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + class_weight=None, + best_parameters=None, + random_state=None): + if max_depth == "None" or max_depth is None or max_depth == 0: + max_depth = None + if class_weight == "None" or class_weight is None: + class_weight = None + + best_min_sample_split = best_parameters['model__min_sample_spllit'] + best_min_sample_split_leaf = best_parameters['model__min_sample_leaf'] + + model = DecisionTreeClassifier( + criterion=criterion, + max_depth=max_depth, + min_samples_split=best_min_sample_split, + min_samples_leaf=best_min_sample_split_leaf, + class_weight=class_weight, + random_state=random_state + ) + return model + + def VS_naive_bayes_Get_best_params(self): + model = GaussianNB() + + return model + + def Vs_svm_Get_best_params(self, + C=1.0, + kernel='rbf', + degree=3, + coef0=0.0, + # cache_size=200.0, + class_wight=None, + gamma="scale", + decision_function_shape="ovr", + # verbos=False, + # max_iter=1, + best_parameters=None + ): + if class_wight == "None": + class_wight = None + # if verbos == "False": + # verbos = False + # else: + # verbos = True + + best_c = best_parameters['model__c'] + best_kernel = best_parameters['model__kernel'], + best_degree = best_parameters['model__degree'], + best_coefo = best_parameters['model__ceofo'], + best_cache_size = best_parameters['model__cache_size'], + best_class_weight = best_parameters['model__class_weight'], + best_verbos = best_parameters['model__verbos'], + best_max_iter = best_parameters['model__max_iter'], + + model = SVC( + C=best_c, + kernel=best_kernel, + degree=best_degree, + coef0=best_coefo, + # cache_size=best_cache_size, + class_weight=best_class_weight, + gamma=gamma, + decision_function_shape=decision_function_shape, + # verbose=best_verbos, + # max_iter=best_max_iter + ) + + return model + + def Vs_decision_tree_classifier(self, + criterion='gini', + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + class_weight=None, + random_state=None): + if max_depth == "None" or max_depth is None or max_depth == 0: + max_depth = None + + if class_weight == "None" or class_weight is None: + class_weight = None + + model = DecisionTreeClassifier( + criterion=criterion, + max_depth=max_depth, + min_samples_split=min_samples_split, + min_samples_leaf=min_samples_leaf, + class_weight=class_weight, + random_state=random_state + ) + self.model = model + y_train_pred, y_val_pred, y_test_pred = self.Vs_fit_and_predict_model() + train_score_dict, val_score_dict, test_score_dict = self.Vs_Score_cls_All_Data(y_train_pred, y_val_pred, + y_test_pred) + + return {'x_train': self.X_train, 'x_val': self.X_val, 'x_test': self.X_test, + 'y_train_pred': y_train_pred, 'y_val_pred': y_val_pred, 'y_test_pred': y_test_pred, + 'y_train': np.array(self.y_train), 'y_val': np.array(self.y_val), 'y_test': np.array(self.y_test), + 'train_score_dict': train_score_dict, 'val_score_dict': val_score_dict, + 'test_score_dict': test_score_dict, 'model_params': model.get_params()} + + def Vs_naive_bayes_classifier(self): + model = GaussianNB() + + self.model = model + y_train_pred, y_val_pred, y_test_pred = self.Vs_fit_and_predict_model() + train_score_dict, val_score_dict, test_score_dict = self.Vs_Score_cls_All_Data(y_train_pred, y_val_pred, + y_test_pred) + + return {'x_train': self.X_train, 'x_val': self.X_val, 'x_test': self.X_test, + 'y_train_pred': y_train_pred, 'y_val_pred': y_val_pred, 'y_test_pred': y_test_pred, + 'y_train': np.array(self.y_train), 'y_val': np.array(self.y_val), 'y_test': np.array(self.y_test), + 'train_score_dict': train_score_dict, 'val_score_dict': val_score_dict, + 'test_score_dict': test_score_dict, 'model_params': model.get_params()} + + def Vs_svm_classifier(self, + C=1.0, + kernel='rbf', + degree=3, + coef0=0.0, + # cache_size=200.0, + class_wight=None, + gamma="scale", + decision_function_shape="ovr" + # verbos=False, + # max_iter=1, + # random_state=None + ): + model = SVC( + C=C, + kernel=kernel, + degree=degree, + coef0=coef0, + # cache_size=cache_size, + class_weight=class_wight, + gamma=gamma, + decision_function_shape=decision_function_shape + # verbose=verbos, + # max_iter=-max_iter, + # random_state=random_state + ) + + self.model = model + y_train_pred, y_val_pred, y_test_pred = self.Vs_fit_and_predict_model() + train_score_dict, val_score_dict, test_score_dict = self.Vs_Score_cls_All_Data(y_train_pred, y_val_pred, + y_test_pred) + + return {'x_train': self.X_train, 'x_val': self.X_val, 'x_test': self.X_test, + 'y_train_pred': y_train_pred, 'y_val_pred': y_val_pred, 'y_test_pred': y_test_pred, + 'y_train': np.array(self.y_train), 'y_val': np.array(self.y_val), 'y_test': np.array(self.y_test), + 'train_score_dict': train_score_dict, 'val_score_dict': val_score_dict, + 'test_score_dict': test_score_dict, 'model_params': model.get_params()} diff --git a/Galaxy/galaxy/tools/radiuma/preprocessing.py b/Galaxy/galaxy/tools/radiuma/preprocessing.py new file mode 100644 index 0000000..b9ddfa9 --- /dev/null +++ b/Galaxy/galaxy/tools/radiuma/preprocessing.py @@ -0,0 +1,47 @@ +from sklearn.utils import resample, shuffle + +class preprocessing: + + def __init__(self,X): + self.X = X + + + + # def SelectAlg(self): + # import json + # f = open('data.json') + # JSONdata = json.load(f) + # f.close() + # algn = self.AlgName + # if self.AlgName == "resample": + + # self.Vs_resample( + # JSONdata[algn][0], + # JSONdata[algn][1] + # ) + # elif self.AlgName == "shuffle": + # self.Vs_shuffle( + # JSONdata[algn][0], + # JSONdata[algn][1] + # ) + # else: + # print("Algorithm name is wrong") + + + + def Vs_resampling(self,str=False,rep=False): + + if str: + x2 = resample(self.X , replace=rep) + else: + x2 = resample(self.X , replace=rep) + + return x2 + + + + def Vs_shuffling(self): + + x2 = shuffle(self.X) + + return x2 diff --git a/Galaxy/galaxy/tools/radiuma/preprocessing.xml b/Galaxy/galaxy/tools/radiuma/preprocessing.xml new file mode 100644 index 0000000..7593baa --- /dev/null +++ b/Galaxy/galaxy/tools/radiuma/preprocessing.xml @@ -0,0 +1,13 @@ + + python3 $__tool_directory__/preprocessing.py --input $input --method $method --output $output + + + + + + + + + + + diff --git a/Galaxy/galaxy/tools/radiuma/radiomics.py b/Galaxy/galaxy/tools/radiuma/radiomics.py new file mode 100644 index 0000000..63ebf91 --- /dev/null +++ b/Galaxy/galaxy/tools/radiuma/radiomics.py @@ -0,0 +1,556 @@ +import tkinter as tk +from tkinter import ttk, messagebox +from tkinter.filedialog import askopenfilename, askdirectory +from PIL import Image, ImageTk +from prompt_toolkit.data_structures import Point + + +def radiomics_feature_generator_view(self, oid): + def _validate(P, mi, ma): + if P == "": + P = mi + if P.count(".") > 1: + return False + P = P.replace(".", "") + return P.isdigit() and float(P) >= float(mi) and float(P) <= float(ma) + + def _validate_c(P, mi, ma): + if P == "": + P = mi + items = P.split(",") + for i in items: + if i.count(".") > 1: + return False + j = i.replace(".", "") + if j == '': + j = '0' + i = '0.' + if not (j.isdigit() and float(i) >= float(mi) and float(i) <= float(ma)) and i != "": + return False + return True + + def _validate_int(P, mi, ma): + if P == "": + P = mi + try: + if P == '-': + P = '-1' + return int(P) >= int(mi) and int(P) <= int(ma) + except: + return False + + self.arr_img = ImageTk.PhotoImage(Image.open("images/arrow-icon.png").resize((20, 20))) + main = self.newWindow + main.none = False + main.attributes('-toolwindow', True) + main.title('Radiomic Feature Generator') + main.resizable(False, False) + + self.rbg = "#FFFFFF" + root = tk.Frame(main, bg=self.rbg) + # menu = tk.Frame(main, bg=self.bg) + root.grid(row=0, column=1) + + # first row + # ttk.Separator(root, orient='horizontal').grid(column=0, row=0, columnspan=7, sticky="ew", padx=5, pady=5) + frame = tk.Frame(root, highlightthickness=2, highlightcolor="#D1CFE2", bg=self.rbg) + frame.grid(column=0, row=0, columnspan=6, sticky="news", padx=16, pady=(10, 5), ipady=2, ipadx=2) + tk.Label(frame, text="Source", bg=self.rbg, fg="#1C1C28").grid(column=0, row=0, sticky=tk.W, padx=5, pady=5) + + self.ra_file_folder = tk.StringVar(value=0) + self.ra_file_folder.set(self.ras_file_folder[oid].get()) + + def select_radio(clean=True): + if self.ra_file_folder.get() == "0": + if clean: + if self.comp_conn[oid] == -1: + self.ra_file_slc_lbl.set("Select a file...") + self.ra_file_slc2_lbl.set("Select a file...") + elif self.comp_conn[oid] == 0 or self.comp_conn[oid] == 2: + self.ra_file_slc_lbl.set("Set by connection link") + self.ra_file_slc2_lbl.set("Select a file...") + else: + self.ra_file_slc_lbl.set("Select a file...") + self.ra_file_slc2_lbl.set("Set by connection link") + self.file_select = "" + self.file_slc_btn.config(image=self.slc_file_photo) + self.file_slc2_btn.config(image=self.slc_file_photo) + else: + if clean: + if self.comp_conn[oid] == -1: + self.ra_file_slc_lbl.set("Select a folder...") + self.ra_file_slc2_lbl.set("Select a folder...") + elif self.comp_conn[oid] == 0 or self.comp_conn[oid] == 2: + self.ra_file_slc_lbl.set("Set by connection link") + self.ra_file_slc2_lbl.set("Select a folder...") + else: + self.ra_file_slc_lbl.set("Select a folder...") + self.ra_file_slc2_lbl.set("Set by connection link") + self.file_select = "" + self.file_slc_btn.config(image=self.slc_folder_photo) + self.file_slc2_btn.config(image=self.slc_folder_photo) + + tk.Radiobutton(frame, bg=self.rbg, text="Single file", variable=self.ra_file_folder, value=0, + command=select_radio).grid(column=1, row=0, padx=(0, 0)) + tk.Radiobutton(frame, bg=self.rbg, text="Folder", variable=self.ra_file_folder, value=1, command=select_radio).grid( + column=1, row=0, padx=(150, 0)) + tk.Label(frame, text="Original image", bg=self.rbg).grid(column=0, row=2, sticky=tk.W, padx=(5, 26), pady=5) + self.ra_file_slc_lbl = tk.StringVar(value="Select a file...") + if self.comp_conn[oid] == 0 or self.comp_conn[oid] == 2: + self.ra_file_slc_lbl.set("Set by connection link") + else: + t = self.ras_file_slc_lbl[oid].get() + if t.startswith("Set"): + if self.ras_file_folder[oid].get() == "0": + t = "Select a file..." + else: + t = "Select a folder..." + self.ra_file_slc_lbl.set(t) + ttk.Entry(frame, textvariable=self.ra_file_slc_lbl, state="disabled").grid(column=0, row=2, sticky="we", + padx=(100, 0), pady=5, ipady=10) + + def select_file(): + self.newWindow.attributes('-topmost', True) + if self.ra_file_folder.get() == "0": + self.file_select = askopenfilename(parent=root) + else: + self.file_select = askdirectory(parent=root) + self.newWindow.attributes('-topmost', False) + self.ra_file_slc_lbl.set(self.file_select) + + self.slc_file_photo = tk.PhotoImage(file="images/file.png") + self.slc_folder_photo = tk.PhotoImage(file="images/folder.png") + self.file_slc_btn = tk.Button(frame, image=self.slc_file_photo, command=select_file, bg="#9CADCE", borderwidth=0, + height=40, width=40, state='disabled' if ( + self.comp_conn[oid] == 0 or self.comp_conn[oid] == 2) else 'normal') + self.file_slc_btn.grid(column=0, row=2, sticky="w", padx=(220, 0)) + + tk.Label(frame, text="Region of interest (ROI)", bg=self.rbg).grid(column=1, row=2, sticky="e", padx=5, pady=5) + self.ra_file_slc2_lbl = tk.StringVar(value="Select a file...") + if self.comp_conn[oid] == 1 or self.comp_conn[oid] == 2: + self.ra_file_slc2_lbl.set("Set by connection link") + else: + t = self.ras_file_slc2_lbl[oid].get() + if t.startswith("Set"): + if self.ras_file_folder[oid].get() == "0": + t = "Select a file..." + else: + t = "Select a folder..." + self.ra_file_slc2_lbl.set(t) + ttk.Entry(frame, textvariable=self.ra_file_slc2_lbl, state="disabled").grid(column=2, row=2, sticky="we", + padx=(5, 0), pady=5, columnspan=2, + ipady=10, ipadx=4) + + def select_file2(): + self.newWindow.attributes('-topmost', True) + if self.ra_file_folder.get() == "0": + self.file_select2 = askopenfilename(parent=root) + else: + self.file_select2 = askdirectory(parent=root) + self.newWindow.attributes('-topmost', False) + self.ra_file_slc2_lbl.set(self.file_select2) + + self.file_slc2_btn = tk.Button(frame, image=self.slc_file_photo, command=select_file2, bg="#9CADCE", borderwidth=0, + height=40, width=40, state='disabled' if ( + self.comp_conn[oid] == 1 or self.comp_conn[oid] == 2) else 'normal') + self.file_slc2_btn.grid(column=7, row=2, sticky="w") + select_radio(False) + + # ttk.Separator(root, orient='horizontal').grid(column=0, row=4, columnspan=7, sticky="ew", padx=5, pady=5) + frame = tk.Frame(root, highlightthickness=2, highlightcolor="#D1CFE2", bg=self.rbg) + frame.grid(column=0, row=4, columnspan=6, sticky="news", padx=16, pady=5, ipady=2, ipadx=2) + tk.Label(frame, text="Destination", bg=self.rbg, fg="#1C1C28").grid(column=0, row=4, sticky=tk.W, padx=5, pady=5) + self.ra_file_dest_lbl = tk.StringVar(value="Select a folder...") + self.ra_file_dest_lbl.set(self.ras_file_dest_lbl[oid].get()) + ttk.Entry(frame, textvariable=self.ra_file_dest_lbl, state="disabled").grid(column=1, row=4, sticky="we", + padx=(5, 0), pady=5, columnspan=2, + ipady=10, ipadx=4) + + def select_dest_folder(): + self.newWindow.attributes('-topmost', True) + self.dest_select = askdirectory(parent=root) + self.newWindow.attributes('-topmost', False) + self.ra_file_dest_lbl.set(self.dest_select) + + tk.Button(frame, image=self.slc_folder_photo, command=select_dest_folder, bg="#9CADCE", borderwidth=0, height=40, + width=40, ).grid(column=3, row=4, sticky="w") + + # second tab + # ttk.Separator(root, orient='horizontal').grid(column=0, row=6, columnspan=7, sticky="ew", padx=5, pady=5) + frame = tk.Frame(root, highlightthickness=2, highlightcolor="#D1CFE2", bg=self.rbg) + frame.grid(column=0, row=6, columnspan=6, sticky="news", padx=16, pady=5, ipady=2, ipadx=2) + tk.Label(frame, text="Parameters", bg=self.rbg, fg="#1C1C28").grid(column=0, row=6, sticky=tk.W, padx=5, pady=5) + tk.Label(frame, bg=self.rbg, + text="Image modality type").grid(column=0, + row=7, + padx=3, + pady=3, sticky="w") + self.ra_rfg_imt_value = tk.StringVar(value="CT") + self.ra_rfg_imt_value.set(self.ras_rfg_imt_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_imt_value, "CT", "PET", "SPECT", "MR") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=1, row=7, padx=3, pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, + text="Intensity outlier re-segmentatin flag").grid(column=2, row=7, padx=(20, 3), pady=3, sticky="w") + self.ra_rfg_iorf_value = tk.StringVar(value="0") + self.ra_rfg_iorf_value.set(self.ras_rfg_iorf_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_iorf_value, "0", "1") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=3, + row=7, + padx=3, + pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, + text="Discretization type").grid(column=0, + row=8, + padx=3, + pady=3, sticky="w") + self.ra_rfg_dit_value = tk.StringVar(value="FBN") + self.ra_rfg_dit_value.set(self.ras_rfg_dit_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_dit_value, "FBN", "FBS") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=1, + row=8, + padx=3, + pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, + text="Image quantization flag").grid(column=2, row=8, padx=(20, 3), pady=3, sticky="w") + self.ra_rfg_iqf_value = tk.StringVar(value="0") + self.ra_rfg_iqf_value.set(self.ras_rfg_iqf_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_iqf_value, "0", "1") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=3, + row=8, + padx=3, + pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, + text="Bin size/width").grid(column=0, row=9, padx=3, pady=3, sticky="w") + self.ra_rgd_nhb_value = tk.StringVar(value="8") + self.ra_rgd_nhb_value.set(self.ras_rgd_nhb_value[oid].get()) + ttk.Entry(frame, textvariable=self.ra_rgd_nhb_value, validate="all", justify="center", + validatecommand=(frame.register(_validate_c), "%P", 0, 128)).grid(column=1, row=9, padx=3, pady=3, + sticky="we") + + tk.Label(frame, bg=self.rbg, text="Re-segmentation interval range").grid(column=2, row=9, padx=(20, 2), pady=3, + sticky="w") + + tk.Label(frame, bg=self.rbg, text="[").grid(column=3, row=9, padx=(0, 0), pady=3, sticky="w") + self.ra_rgd_rir1_value = tk.StringVar(value=-3000) + self.ra_rgd_rir1_value.set(self.ras_rgd_rir1_value[oid].get()) + ttk.Spinbox(frame, from_=-3000, to=0, textvariable=self.ra_rgd_rir1_value, validate="key", width=6, + validatecommand=(frame.register(_validate_int), "%P", -3000, 0), justify="center").grid(column=3, row=9, + padx=(0, 80), + pady=3, + sticky="e") + tk.Label(frame, bg=self.rbg, text=",").grid(column=3, row=9, padx=(70, 0), pady=3, sticky="w") + self.ra_rgd_rir2_value = tk.StringVar(value=3000) + self.ra_rgd_rir2_value.set(self.ras_rgd_rir2_value[oid].get()) + ttk.Spinbox(frame, from_=0, to=3000, textvariable=self.ra_rgd_rir2_value, validate="key", width=6, + validatecommand=(frame.register(_validate_int), "%P", 0, 3000), justify="center").grid(column=3, row=9, + padx=(80, 0), + pady=3, + sticky="w") + tk.Label(frame, bg=self.rbg, text="]").grid(column=3, row=9, padx=(140, 0), pady=0, sticky="w") + + tk.Label(frame, bg=self.rbg, text="Resampling (scaling) flag").grid(column=0, row=10, padx=3, pady=3, sticky="w") + self.ra_rfg_rf_value = tk.StringVar(value="1") + self.ra_rfg_rf_value.set(self.ras_rfg_rf_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_rf_value, "0", "1") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=1, row=10, padx=3, pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, text="ROI partial volume threshold").grid(column=2, row=10, padx=(20, 3), pady=3, + sticky="w") + self.ra_rgd_roi_value = tk.StringVar(value=0.5) + self.ra_rgd_roi_value.set(self.ras_rgd_roi_value[oid].get()) + ttk.Spinbox(frame, from_=0, to=1, increment=0.1, textvariable=self.ra_rgd_roi_value, validate="key", + validatecommand=(frame.register(_validate), "%P", 0, 1), justify="center").grid(column=3, row=10, + padx=3, pady=3, + sticky="we") + tk.Label(frame, bg=self.rbg, text="Image resampling interpolation type").grid(column=0, row=11, padx=3, pady=3, + sticky="w") + self.ra_rfg_irit_value = tk.StringVar(value="Nearest") + self.ra_rfg_irit_value.set(self.ras_rfg_irit_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_irit_value, "Nearest", "Linear", "Cubic") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=1, row=11, padx=3, pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, text="Quantization type").grid(column=2, row=11, padx=(20, 3), pady=3, sticky="w") + self.ra_rfg_qt_value = tk.StringVar(value="Uniform") + self.ra_rfg_qt_value.set(self.ras_rfg_qt_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_qt_value, "Uniform") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=3, row=11, padx=3, pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, text="ROI resampling interpolation type").grid(column=0, row=12, padx=3, pady=3, + sticky="w") + self.ra_rfg_roit_value = tk.StringVar(value="Nearest") + self.ra_rfg_roit_value.set(self.ras_rfg_roit_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_roit_value, "Nearest", "Linear", "Cubic") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=1, row=12, padx=3, pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, text="Intensity volume histogram (IVH) type").grid(column=2, row=12, padx=(20, 3), + pady=3, sticky="w") + self.ra_rfg_ivht_value = tk.StringVar(value="1") + self.ra_rfg_ivht_value.set(self.ras_rfg_ivht_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_ivht_value, "0", "1", "2", "3") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=3, row=12, padx=3, pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, text="3D isotropic voxel size flag (mm)").grid(column=0, row=13, padx=3, pady=3, + sticky="w") + self.ra_rgd_3ivs_value = tk.StringVar(value=6.00) + self.ra_rgd_3ivs_value.set(self.ras_rgd_3ivs_value[oid].get()) + ttk.Spinbox(frame, from_=0, to=50, increment=1, textvariable=self.ra_rgd_3ivs_value, validate="key", + validatecommand=(frame.register(_validate), "%P", 0, 50), justify="center").grid(column=1, row=13, + padx=3, pady=3, + sticky="we") + tk.Label(frame, bg=self.rbg, text="IVH discretization type").grid(column=2, row=13, padx=(20, 3), pady=3, + sticky="w") + self.ra_rfg_idt_value = tk.StringVar(value="0") + self.ra_rfg_idt_value.set(self.ras_rfg_idt_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_idt_value, "0", "1") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=3, row=13, padx=3, pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, text="2D isotropic voxel size flag (mm)").grid(column=0, + row=14, + padx=3, + pady=3, sticky="w") + self.ra_rgd_2ivs_value = tk.StringVar(value=1.00) + self.ra_rgd_2ivs_value.set(self.ras_rgd_2ivs_value[oid].get()) + ttk.Spinbox(frame, from_=0, to=50, increment=1, textvariable=self.ra_rgd_2ivs_value, validate="key", + validatecommand=(frame.register(_validate), "%P", 0, 50), justify="center").grid(column=1, + row=14, + padx=3, + pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, + text="IVH discretization binning option (number/width)").grid(column=2, row=14, padx=(20, 3), pady=3, + sticky="w") + self.ra_rgd_ivho_value = tk.StringVar(value=200.00) + self.ra_rgd_ivho_value.set(self.ras_rgd_ivho_value[oid].get()) + ttk.Spinbox(frame, from_=0, to=1000, increment=1, textvariable=self.ra_rgd_ivho_value, validate="key", + validatecommand=(frame.register(_validate), "%P", 0, 1000), justify="center").grid(column=3, + row=14, + padx=3, + pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, + text="Isotropic 2D voxels flag").grid(column=0, + row=15, + padx=3, + pady=3, sticky="w") + self.ra_rfg_i2vf_value = tk.StringVar(value="0") + self.ra_rfg_i2vf_value.set(self.ras_rfg_i2vf_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_i2vf_value, "0", "1") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=1, + row=15, + padx=3, + pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, + text="Max number of ROIs per image").grid(column=2, row=15, padx=(20, 3), pady=3, sticky="w") + self.ra_rgd_mri_value = tk.StringVar(value="4") + self.ra_rgd_mri_value.set(self.ras_rgd_mri_value[oid].get()) + ttk.Spinbox(frame, from_=0, to=1000000, textvariable=self.ra_rgd_mri_value, validate="key", + validatecommand=(frame.register(_validate_int), "%P", 0, 1000000), justify="center").grid(column=3, + row=15, + padx=3, + pady=3, + sticky="we") + tk.Label(frame, bg=self.rbg, + text="Round voxel intensity values flag").grid(column=0, + row=16, + padx=3, + pady=3, sticky="w") + self.ra_rfg_rviv_value = tk.StringVar(value="0") + self.ra_rfg_rviv_value.set(self.ras_rfg_rviv_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_rviv_value, "0", "1") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=1, + row=16, + padx=3, + pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, + text="Combine multiple ROIs to one flag").grid(column=2, row=16, padx=(20, 3), pady=3, sticky="w") + self.ra_rfg_cmrf_value = tk.StringVar(value="0") + self.ra_rfg_cmrf_value.set(self.ras_rfg_cmrf_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_cmrf_value, "0", "1") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=3, + row=16, + padx=3, + pady=3, sticky="we") + tk.Label(frame, bg=self.rbg, + text="Range re-segmentation flag").grid(column=0, + row=17, + padx=3, + pady=3, sticky="w") + self.ra_rfg_rrf_value = tk.StringVar(value="0") + self.ra_rfg_rrf_value.set(self.ras_rfg_rrf_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_rrf_value, "0", "1") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=1, + row=17, + padx=3, + pady=3, sticky="we") + tk.Label(frame, text="Type of output data", bg=self.rbg).grid(column=2, row=17, padx=(20, 3), pady=3, sticky="w") + self.ra_rfg_tod_value = tk.StringVar(value="2") + self.ra_rfg_tod_value.set(self.ras_rfg_tod_value[oid].get()) + w = tk.OptionMenu(frame, self.ra_rfg_tod_value, "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12") + w.config(indicatoron=False, compound='right', image=self.arr_img) + w.config(highlightthickness=0, highlightbackground="black") + w.grid(column=3, + row=17, + padx=3, + pady=3, sticky="we") + + # last row + # self.close_photo = tk.PhotoImage(file="images/close.png") + # self.close_photo = self.close_photo.zoom(5) + # self.close_photo = self.close_photo.subsample(3) + + def destroy(): + self.set_radiomix_params(oid) + root.quit() + root.destroy() + main.none = True + + def close(): + main.quit() + main.destroy() + # root.quit() + # root.destroy() + + main.protocol("WM_DELETE_WINDOW", close) + # root.protocol("WM_DELETE_WINDOW", destroy) + # ttk.Button(root, image=self.close_photo, command=destroy).grid(column=3, row=18, sticky="e", padx=40) + tk.Button(root, image=self.close_photo, bg=self.rbg, command=destroy, borderwidth=0).grid(column=1, row=18, + columnspan=5, padx=124, + sticky="e", pady=(0, 5)) + + # self.ok_photo = tk.PhotoImage(file="images/OK 25.png") + + # self.ok_photo = self.ok_photo.zoom(3) + # self.ok_photo = self.ok_photo.subsample(2) + def okf(): + for it in self.ra_rgd_nhb_value.get().split(","): + if float(it) < 0 or float(it) > 128: + messagebox.showerror(title="Parameter range", + message="please select 'Bin size/width' paramter in [0, 128] range.") + return + if float(self.ra_rgd_rir1_value.get()) < -3000 or float(self.ra_rgd_rir1_value.get()) > 0: + messagebox.showerror(title="Parameter range", + message="please select 'Re-segmentation interval range' paramter in [-3000, 3000] range.") + elif float(self.ra_rgd_rir2_value.get()) < 0 or float(self.ra_rgd_rir2_value.get()) > 3000: + messagebox.showerror(title="Parameter range", + message="please select 'Re-segmentation interval range' paramter in [-3000, 3000] range.") + elif float(self.ra_rgd_roi_value.get()) < 0 or float(self.ra_rgd_roi_value.get()) > 1: + messagebox.showerror(title="Parameter range", + message="please select 'ROI partial volume threshold' paramter in [0, 1] range.") + elif float(self.ra_rgd_3ivs_value.get()) < 0 or float(self.ra_rgd_3ivs_value.get()) > 50: + messagebox.showerror(title="Parameter range", + message="please select '3D isotropic voxel size flag(mm)' paramter in [0, 50] range.") + elif float(self.ra_rgd_2ivs_value.get()) < 0 or float(self.ra_rgd_2ivs_value.get()) > 50: + messagebox.showerror(title="Parameter range", + message="please select '2D isotropic voxel size flag(mm)' paramter in [0, 50] range.") + elif float(self.ra_rgd_ivho_value.get()) < 0 or float(self.ra_rgd_ivho_value.get()) > 1000: + messagebox.showerror(title="Parameter range", + message="please select 'IVH discretization binning option (number/width)' paramter in [0, 1000] range.") + elif float(self.ra_rgd_mri_value.get()) < 0 or float(self.ra_rgd_mri_value.get()) > 1000: + messagebox.showerror(title="Parameter range", + message="please select 'Max no. of ROIs per image' paramter in [0, 1000] range.") + else: + change = False + change = self.check_entry(self.ras_rfg_tod_value[oid], self.ra_rfg_tod_value, change) + change = self.check_entry(self.ras_rfg_rrf_value[oid], self.ra_rfg_rrf_value, change) + change = self.check_entry(self.ras_rfg_cmrf_value[oid], self.ra_rfg_cmrf_value, change) + change = self.check_entry(self.ras_rfg_rviv_value[oid], self.ra_rfg_rviv_value, change) + change = self.check_entry(self.ras_rgd_mri_value[oid], self.ra_rgd_mri_value, change) + change = self.check_entry(self.ras_rfg_i2vf_value[oid], self.ra_rfg_i2vf_value, change) + change = self.check_entry(self.ras_rgd_ivho_value[oid], self.ra_rgd_ivho_value, change) + change = self.check_entry(self.ras_rgd_2ivs_value[oid], self.ra_rgd_2ivs_value, change) + change = self.check_entry(self.ras_rfg_idt_value[oid], self.ra_rfg_idt_value, change) + change = self.check_entry(self.ras_rgd_3ivs_value[oid], self.ra_rgd_3ivs_value, change) + change = self.check_entry(self.ras_rfg_ivht_value[oid], self.ra_rfg_ivht_value, change) + change = self.check_entry(self.ras_rfg_roit_value[oid], self.ra_rfg_roit_value, change) + change = self.check_entry(self.ras_rfg_qt_value[oid], self.ra_rfg_qt_value, change) + change = self.check_entry(self.ras_rfg_irit_value[oid], self.ra_rfg_irit_value, change) + change = self.check_entry(self.ras_rgd_roi_value[oid], self.ra_rgd_roi_value, change) + change = self.check_entry(self.ras_rfg_rf_value[oid], self.ra_rfg_rf_value, change) + change = self.check_entry(self.ras_rgd_rir2_value[oid], self.ra_rgd_rir2_value, change) + change = self.check_entry(self.ras_rgd_rir1_value[oid], self.ra_rgd_rir1_value, change) + change = self.check_entry(self.ras_rgd_nhb_value[oid], self.ra_rgd_nhb_value, change) + change = self.check_entry(self.ras_rfg_iqf_value[oid], self.ra_rfg_iqf_value, change) + change = self.check_entry(self.ras_rfg_dit_value[oid], self.ra_rfg_dit_value, change) + change = self.check_entry(self.ras_rfg_iorf_value[oid], self.ra_rfg_iorf_value, change) + change = self.check_entry(self.ras_rfg_imt_value[oid], self.ra_rfg_imt_value, change) + change = self.check_entry(self.ras_file_dest_lbl[oid], self.ra_file_dest_lbl, change) + change = self.check_entry(self.ras_file_slc2_lbl[oid], self.ra_file_slc2_lbl, change) + change = self.check_entry(self.ras_file_slc_lbl[oid], self.ra_file_slc_lbl, change) + change = self.check_entry(self.ras_file_folder[oid], self.ra_file_folder, change) + if change: + self.set_object_state(oid, 'normal', 'hidden', 'hidden', 'hidden') + self.check_connection_remove(oid) + root.quit() + root.destroy() + + # ttk.Button(root, image=self.ok_photo, command=okf).grid(column=3, row=18, sticky="e") + tk.Button(root, image=self.ok_img, bg=self.rbg, command=okf, borderwidth=0).grid(column=5, row=18, sticky="e", + padx=(0, 16), pady=(0, 5)) + root.mainloop() + if main.none: + return None + if not (self.ra_file_slc_lbl.get().startswith("Select ") or self.ra_file_slc2_lbl.get().startswith( + "Select ") or self.ra_file_dest_lbl.get().startswith("Select ")): + return True + return False + + +class RoundedButton(tk.Canvas): + def __init__(self, parent, width, height, corner_radius, padding, color, bg, command=None, text=""): + tk.Canvas.__init__(self, parent, borderwidth=0, + relief="flat", highlightthickness=0, bg=bg) + self.command = command + + rad = 2 * corner_radius + + def shape(): + self.create_polygon((padding, height - corner_radius - padding, padding, corner_radius + padding, + padding + corner_radius, padding, width - padding - corner_radius, padding, + width - padding, corner_radius + padding, width - padding, + height - corner_radius - padding, width - padding - corner_radius, height - padding, + padding + corner_radius, height - padding), fill=color, outline=color) + self.create_arc((padding, padding + rad, padding + rad, padding), start=90, extent=90, fill=color, + outline=color) + self.create_arc((width - padding - rad, padding, width - padding, padding + rad), start=0, extent=90, + fill=color, outline=color) + self.create_arc((width - padding, height - rad - padding, width - padding - rad, height - padding), + start=270, extent=90, fill=color, outline=color) + self.create_arc((padding, height - padding - rad, padding + rad, height - padding), start=180, extent=90, + fill=color, outline=color) + self.create_text(width / 2, height / 2, text=text) + + id = shape() + (x0, y0, x1, y1) = self.bbox("all") + width = (x1 - x0) + height = (y1 - y0) + self.configure(width=width, height=height) + self.bind("", self._on_press) + self.bind("", self._on_release) + + def _on_press(self, event): + self.configure(relief="sunken") + + def _on_release(self, event): + self.configure(relief="raised") + if self.command is not None: + self.command() diff --git a/Galaxy/galaxy/tools/radiuma/radiomics.xml b/Galaxy/galaxy/tools/radiuma/radiomics.xml new file mode 100644 index 0000000..f62467a --- /dev/null +++ b/Galaxy/galaxy/tools/radiuma/radiomics.xml @@ -0,0 +1,25 @@ + + Extract IBSI-compliant radiomics features using PySERA + + python3 $__tool_directory__/run_radiomics.py + --image $image + --mask $mask + --output $output.extra_files_path + --categories "$categories" + --dimensions "$dimensions" + --apply_preprocessing "$apply_preprocessing" + + + + + + + + + + + + + This tool extracts handcrafted radiomics features from medical images using the PySERA library. + + diff --git a/Galaxy/galaxy/tools/radiuma/run_classifiers.py b/Galaxy/galaxy/tools/radiuma/run_classifiers.py new file mode 100644 index 0000000..b21f53d --- /dev/null +++ b/Galaxy/galaxy/tools/radiuma/run_classifiers.py @@ -0,0 +1,21 @@ +from pysera.classifier import Classifier +import argparse +import pandas as pd + +parser = argparse.ArgumentParser() +parser.add_argument("--x_train", required=True) +parser.add_argument("--y_train", required=True) +parser.add_argument("--x_val", required=True) +parser.add_argument("--y_val", required=True) +parser.add_argument("--algorithm", choices=["LogisticRegression", "KNeighborsClassifier"], required=True) +parser.add_argument("--output", required=True) +args = parser.parse_args() + +X_train = pd.read_csv(args.x_train) +y_train = pd.read_csv(args.y_train) +X_val = pd.read_csv(args.x_val) +y_val = pd.read_csv(args.y_val) + +clf = Classifier(X_train, y_train, X_val, y_val) +result = clf.run(algorithm=args.algorithm) +pd.DataFrame(result).to_csv(args.output, index=False) diff --git a/Galaxy/galaxy/tools/radiuma/run_preprocessing.py b/Galaxy/galaxy/tools/radiuma/run_preprocessing.py new file mode 100644 index 0000000..0312ce1 --- /dev/null +++ b/Galaxy/galaxy/tools/radiuma/run_preprocessing.py @@ -0,0 +1,19 @@ +from pysera.preprocessing import Preprocessor +import argparse +import pandas as pd + +parser = argparse.ArgumentParser() +parser.add_argument("--input", required=True) +parser.add_argument("--method", choices=["resample", "shuffle"], required=True) +parser.add_argument("--output", required=True) +args = parser.parse_args() + +df = pd.read_csv(args.input) +prep = Preprocessor(df) + +if args.method == "resample": + result = prep.resample() +elif args.method == "shuffle": + result = prep.shuffle() + +result.to_csv(args.output, index=False) diff --git a/Galaxy/galaxy/tools/radiuma/run_radiomics.py b/Galaxy/galaxy/tools/radiuma/run_radiomics.py new file mode 100644 index 0000000..a196ff4 --- /dev/null +++ b/Galaxy/galaxy/tools/radiuma/run_radiomics.py @@ -0,0 +1,34 @@ +# run_radiomics.py + +import pysera +import argparse +import os + +parser = argparse.ArgumentParser() +parser.add_argument("--image", required=True) +parser.add_argument("--mask", required=True) +parser.add_argument("--output", required=True) +parser.add_argument("--categories", default="glcm, glrlm") +parser.add_argument("--dimensions", default="1st, 2_5d, 3d") +parser.add_argument("--apply_preprocessing", default="True") +args = parser.parse_args() + +# اطمینان از وجود مسیر خروجی +os.makedirs(args.output, exist_ok=True) + +# اجرای PySERA +result = pysera.process_batch( + image_input=args.image, + mask_input=args.mask, + output_path=args.output, + categories=args.categories, + dimensions=args.dimensions, + apply_preprocessing=args.apply_preprocessing == "True" +) + +# ذخیره خروجی ویژگی‌ها به CSV +if result["success"]: + features = result["features_extracted"] + features.to_csv(os.path.join(args.output, "radiomics_features.csv"), index=False) +else: + print("Error:", result.get("error", "Unknown error")) diff --git a/Galaxy/pysera/__init__.py b/Galaxy/pysera/__init__.py new file mode 100644 index 0000000..ac49316 --- /dev/null +++ b/Galaxy/pysera/__init__.py @@ -0,0 +1,3 @@ +from .classifiers import Classifiers +from .preprocessor import Preprocessor +from .radiomics_extractor import RadiomicsExtractor \ No newline at end of file diff --git a/Galaxy/pysera/classifiers.py b/Galaxy/pysera/classifiers.py new file mode 100644 index 0000000..e69de29 diff --git a/Galaxy/pysera/preprocessor.py b/Galaxy/pysera/preprocessor.py new file mode 100644 index 0000000..5714dd9 --- /dev/null +++ b/Galaxy/pysera/preprocessor.py @@ -0,0 +1,12 @@ +import pandas as pd +from sklearn.utils import resample, shuffle + +class Preprocessor: + def __init__(self, df: pd.DataFrame): + self.df = df + + def resample(self, replace=False): + return resample(self.df, replace=replace) + + def shuffle(self): + return shuffle(self.df) \ No newline at end of file diff --git a/Galaxy/pysera/radiomics_extractor.py b/Galaxy/pysera/radiomics_extractor.py new file mode 100644 index 0000000..e69de29 diff --git a/Galaxy/requirements.txt b/Galaxy/requirements.txt new file mode 100644 index 0000000..ebb4a5f --- /dev/null +++ b/Galaxy/requirements.txt @@ -0,0 +1,90 @@ +astroid==4.0.1 +certifi==2025.10.5 +charset-normalizer==3.4.4 +cleo==2.1.0 +colorama==0.4.6 +connected-components-3d==3.26.0 +contourpy==1.3.3 +crashtest==0.4.1 +cycler==0.12.1 +dataclasses==0.6 +dcmrtstruct2nii==5 +dill==0.4.0 +distlib==0.4.0 +docx==0.2.4 +et_xmlfile==2.0.0 +filelock==3.20.0 +fonttools==4.60.1 +fpdf==1.7.2 +fsspec==2025.9.0 +idna==3.11 +imageio==2.37.0 +importlib_resources==6.5.2 +iniconfig==2.3.0 +isort==7.0.0 +itk==5.4.4.post1 +itk-core==5.4.4.post1 +itk-filtering==5.4.4.post1 +itk-io==5.4.4.post1 +itk-numerics==5.4.4.post1 +itk-registration==5.4.4.post1 +itk-segmentation==5.4.4.post1 +Jinja2==3.1.6 +joblib==1.5.2 +kiwisolver==1.4.9 +kmodes==0.12.2 +lazy_loader==0.4 +lxml==6.0.2 +MarkupSafe==3.0.3 +matplotlib==3.10.7 +mccabe==0.7.0 +mpmath==1.3.0 +networkx==3.5 +nibabel==5.3.2 +numpy==2.2.6 +opencv-python==4.12.0.88 +openpyxl==3.1.5 +packaging==25.0 +pandas==2.3.3 +pillow==12.0.0 +platformdirs==4.5.0 +pluggy==1.6.0 +psutil==7.1.2 +psycopg2-binary==2.9.11 +pyaml==25.7.0 +pydicom==3.0.1 +Pygments==2.19.2 +pylint==4.0.1 +pynrrd==1.1.3 +pyparsing==3.2.5 +pysera==2.1.2 +pytest==8.4.2 +python-dateutil==2.9.0.post0 +python-docx==1.2.0 +pytz==2025.2 +PyWavelets==1.9.0 +PyYAML==6.0.3 +RapidFuzz==3.14.1 +ReliefF==0.1.2 +reportlab==4.4.4 +requests==2.32.5 +rt-utils==1.2.7 +scikit-image==0.25.2 +scikit-learn==1.7.2 +scikit-optimize==0.10.2 +scipy==1.16.3 +six==1.17.0 +sklearn-relief==1.0.0b2 +sympy==1.14.0 +termcolor==3.2.0 +threadpoolctl==3.6.0 +tifffile==2025.10.16 +tomlkit==0.13.3 +torch==2.8.0 +torchaudio==2.8.0 +torchvision==0.23.0 +typing_extensions==4.15.0 +tzdata==2025.2 +uml-class-diagram-generator==0.1 +urllib3==2.5.0 +virtualenv==20.35.3 diff --git a/Luigi/README.md b/Luigi/README.md new file mode 100644 index 0000000..a825808 --- /dev/null +++ b/Luigi/README.md @@ -0,0 +1,22 @@ +# Readme For ***Radiuma_Luigi*** + +This section contains research and development (R&D) projects related to the **Luigi** tool. +Luigi is a lightweight and simple workflow engine for building data and scientific pipelines that focuses on reproducibility and dependency management. + +## Project Goals +- Design simple and executable pipelines in the desktop environment +- Review Luigi's capabilities for managing dependencies and parallel execution +- Test the reproducibility of results and recording artifacts for scientific analysis +- Compare Luigi's performance with other tools (such as Galaxy and Dagster) + +## Folder Structure +- `tasks/` → Contains codes related to task definitions +- `examples/` → Simple examples for testing and execution +- `logs/` → Outputs and logs related to pipeline execution +- `README.md` → General description of the project and how to run + +## How to run +1. Install Luigi: +```bash +pip install luigi +python run_pipeline.py RadiumaPipeline --local-scheduler diff --git a/Luigi/data/images/CT_pitch.nii.gz b/Luigi/data/images/CT_pitch.nii.gz new file mode 100644 index 0000000..1a51792 Binary files /dev/null and b/Luigi/data/images/CT_pitch.nii.gz differ diff --git a/Luigi/data/masks/CT_pitch_mask.nii.gz b/Luigi/data/masks/CT_pitch_mask.nii.gz new file mode 100644 index 0000000..b7a2ff1 Binary files /dev/null and b/Luigi/data/masks/CT_pitch_mask.nii.gz differ diff --git a/Luigi/engine/__init__.py b/Luigi/engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Luigi/engine/flags.py b/Luigi/engine/flags.py new file mode 100644 index 0000000..84d5a01 --- /dev/null +++ b/Luigi/engine/flags.py @@ -0,0 +1,20 @@ +import os + +def flags_dir(workspace: str) -> str: + d = os.path.join(workspace, "_flags") + os.makedirs(d, exist_ok=True) + return d + +def flag_path(workspace: str, name: str) -> str: + return os.path.join(flags_dir(workspace), name) + +def is_set(workspace: str, name: str) -> bool: + return os.path.exists(flag_path(workspace, name)) + +def set_flag(workspace: str, name: str) -> None: + open(flag_path(workspace, name), "a").close() + +def clear_flag(workspace: str, name: str) -> None: + p = flag_path(workspace, name) + if os.path.exists(p): + os.remove(p) diff --git a/Luigi/engine/pipeline.py b/Luigi/engine/pipeline.py new file mode 100644 index 0000000..f99628e --- /dev/null +++ b/Luigi/engine/pipeline.py @@ -0,0 +1,19 @@ +import luigi +import time +from engine.tasks_writer import ImageWriter + +class RadiumaPipeline(luigi.WrapperTask): + artifacts_dir = luigi.Parameter(default="artifacts") + + def requires(self): + # start timer at the very beginning + self.start_time = time.time() + return ImageWriter(artifacts_dir=self.artifacts_dir) + + def run(self): + elapsed = time.time() - self.start_time + hours, rem = divmod(elapsed, 3600) + minutes, seconds = divmod(rem, 60) + centiseconds = int((seconds - int(seconds)) * 100) + print(f"[Pipeline] total execution time: {int(hours):02}:{int(minutes):02}:{int(seconds):02}.{centiseconds:02}") + print("=== Workflow completed successfully ===") diff --git a/Luigi/engine/provenance.py b/Luigi/engine/provenance.py new file mode 100644 index 0000000..8b79310 --- /dev/null +++ b/Luigi/engine/provenance.py @@ -0,0 +1,26 @@ +import os, time, json, hashlib +from typing import Iterable + +def sha256_file(fp: str) -> str: + h = hashlib.sha256() + with open(fp, "rb") as f: + for chunk in iter(lambda: f.read(1<<20), b""): + h.update(chunk) + return h.hexdigest() + +def write_sidecar(outputs: Iterable, params: dict) -> None: + outs = list(outputs) + if not outs: return + prov_path = os.path.join(os.path.dirname(outs[0].path), "_provenance.json") + meta = { + "params": params, + "tool": {"name": "Radiuma-Luigi", "version": params.get("tool_version", "0.1.0")}, + "timestamps": {"finished_at": time.strftime("%Y-%m-%d %H:%M:%S")}, + "checksums": {} + } + for o in outs: + p = o.path + if os.path.exists(p): + meta["checksums"][os.path.basename(p)] = sha256_file(p) + with open(prov_path, "w", encoding="utf-8") as f: + json.dump(meta, f, indent=2) diff --git a/Luigi/engine/tasks_conversion.py b/Luigi/engine/tasks_conversion.py new file mode 100644 index 0000000..18369b0 --- /dev/null +++ b/Luigi/engine/tasks_conversion.py @@ -0,0 +1,29 @@ +import os +import luigi +import SimpleITK as sitk +from pathlib import Path + +class ImageConversion(luigi.Task): + artifacts_dir = luigi.Parameter(default="artifacts") + + def requires(self): + from engine.tasks_fusion import ImageFusion + return ImageFusion(artifacts_dir=self.artifacts_dir) + + def output(self): + return luigi.LocalTarget(os.path.join(self.artifacts_dir, "converted_index.txt")) + + def run(self): + fused_index = os.path.join(self.artifacts_dir, "fused_index.txt") + with open(fused_index, "r") as f: + fused_paths = [line.strip() for line in f if line.strip()] + + converted_paths = [] + for img_path in fused_paths: + img = sitk.ReadImage(img_path) + out_path = Path(self.artifacts_dir) / f"converted_{Path(img_path).name}" + sitk.WriteImage(img, str(out_path)) + converted_paths.append(str(out_path)) + + with self.output().open("w") as f: + f.write("\n".join(converted_paths)) diff --git a/Luigi/engine/tasks_filter.py b/Luigi/engine/tasks_filter.py new file mode 100644 index 0000000..62389e3 --- /dev/null +++ b/Luigi/engine/tasks_filter.py @@ -0,0 +1,31 @@ +import os +import luigi +import SimpleITK as sitk +from pathlib import Path + +class ImageFilter(luigi.Task): + artifacts_dir = luigi.Parameter(default="artifacts") + sigma = luigi.FloatParameter(default=1.0) + + def requires(self): + from engine.tasks_conversion import ImageConversion + return ImageConversion(artifacts_dir=self.artifacts_dir) + + def output(self): + return luigi.LocalTarget(os.path.join(self.artifacts_dir, "filtered_index.txt")) + + def run(self): + conv_index = os.path.join(self.artifacts_dir, "converted_index.txt") + with open(conv_index, "r") as f: + conv_paths = [line.strip() for line in f if line.strip()] + + filtered_paths = [] + for img_path in conv_paths: + img = sitk.ReadImage(img_path) + filtered_img = sitk.SmoothingRecursiveGaussian(img, sigma=self.sigma) + out_path = Path(self.artifacts_dir) / f"filtered_{Path(img_path).name}" + sitk.WriteImage(filtered_img, str(out_path)) + filtered_paths.append(str(out_path)) + + with self.output().open("w") as f: + f.write("\n".join(filtered_paths)) diff --git a/Luigi/engine/tasks_fusion.py b/Luigi/engine/tasks_fusion.py new file mode 100644 index 0000000..ae66f39 --- /dev/null +++ b/Luigi/engine/tasks_fusion.py @@ -0,0 +1,36 @@ +import os +import luigi +import numpy as np +import SimpleITK as sitk +from pathlib import Path + +class ImageFusion(luigi.Task): + artifacts_dir = luigi.Parameter(default="artifacts") + + def requires(self): + from engine.tasks_registration import ImageRegistration + return ImageRegistration(artifacts_dir=self.artifacts_dir) + + def output(self): + return luigi.LocalTarget(os.path.join(self.artifacts_dir, "fused_index.txt")) + + def run(self): + reg_index = os.path.join(self.artifacts_dir, "registered_index.txt") + with open(reg_index, "r") as f: + reg_paths = [line.strip() for line in f if line.strip()] + + fused_paths = [] + for img_path in reg_paths: + img = sitk.ReadImage(img_path) + arr = sitk.GetArrayFromImage(img) + p5, p95 = np.percentile(arr, [5, 95]) + arr = np.clip(arr, p5, p95) + arr = (arr - p5) / (p95 - p5) if p95 > p5 else arr * 0.0 + fused_img = sitk.GetImageFromArray(arr) + fused_img.CopyInformation(img) + out_path = Path(self.artifacts_dir) / f"fused_{Path(img_path).name}" + sitk.WriteImage(fused_img, str(out_path)) + fused_paths.append(str(out_path)) + + with self.output().open("w") as f: + f.write("\n".join(fused_paths)) diff --git a/Luigi/engine/tasks_maskreg.py b/Luigi/engine/tasks_maskreg.py new file mode 100644 index 0000000..f6c5e26 --- /dev/null +++ b/Luigi/engine/tasks_maskreg.py @@ -0,0 +1,49 @@ +import os +import luigi +import SimpleITK as sitk +from pathlib import Path + +class MaskRegistration(luigi.Task): + artifacts_dir = luigi.Parameter(default="artifacts") + mask_dir = luigi.Parameter(default=os.path.join("data", "masks")) + + def requires(self): + from engine.tasks_filter import ImageFilter + from engine.tasks_masks import AllMasks + return { + "filter": ImageFilter(artifacts_dir=self.artifacts_dir), + "masks": AllMasks(mask_dir=self.mask_dir) + } + + def output(self): + return luigi.LocalTarget(os.path.join(self.artifacts_dir, "mask_registered_index.txt")) + + def run(self): + filt_index = os.path.join(self.artifacts_dir, "filtered_index.txt") + with open(filt_index, "r") as f: + filtered_paths = [line.strip() for line in f if line.strip()] + + mask_files = [os.path.join(self.mask_dir, f) for f in os.listdir(self.mask_dir) if f.endswith(".nii.gz")] + if not filtered_paths or not mask_files: + raise FileNotFoundError("Missing filtered images or masks for mask_registration.") + + if len(filtered_paths) == len(mask_files): + pairs = zip(filtered_paths, mask_files) + else: + ref_path = filtered_paths[0] + pairs = [(ref_path, m) for m in mask_files] + + out_paths = [] + for ref_img_path, mask_path in pairs: + ref_img = sitk.ReadImage(ref_img_path) + mask_img = sitk.ReadImage(mask_path) + identity = sitk.Transform(ref_img.GetDimension(), sitk.sitkIdentity) + resampled_mask = sitk.Resample( + mask_img, ref_img, identity, sitk.sitkNearestNeighbor, 0, mask_img.GetPixelID() + ) + out_path = Path(self.artifacts_dir) / f"mask_registered_{Path(mask_path).name}" + sitk.WriteImage(resampled_mask, str(out_path)) + out_paths.append(str(out_path)) + + with self.output().open("w") as f: + f.write("\n".join(out_paths)) diff --git a/Luigi/engine/tasks_masks.py b/Luigi/engine/tasks_masks.py new file mode 100644 index 0000000..8c5cd05 --- /dev/null +++ b/Luigi/engine/tasks_masks.py @@ -0,0 +1,17 @@ +import os +import luigi + +class AllMasks(luigi.Task): + mask_dir = luigi.Parameter(default=os.path.join("data", "masks")) + + def output(self): + # Merely as a signal of completion + return luigi.LocalTarget(os.path.join("artifacts", "all_masks.done")) + + def run(self): + files = [os.path.join(self.mask_dir, f) for f in os.listdir(self.mask_dir) if f.endswith(".nii.gz")] + if not files: + raise FileNotFoundError("No masks found in data/masks") + # Just make the done signal. + with self.output().open("w") as f: + f.write(f"{len(files)} masks discovered") diff --git a/Luigi/engine/tasks_processing.py b/Luigi/engine/tasks_processing.py new file mode 100644 index 0000000..ff8506f --- /dev/null +++ b/Luigi/engine/tasks_processing.py @@ -0,0 +1,107 @@ +import luigi +import json +from pathlib import Path +import numpy as np +import SimpleITK as sitk +from engine.tasks_reader import ImageReader +from engine.utils import ensure_dir + +class ImageFusion(luigi.Task): + image_file = luigi.Parameter() + mask_file = luigi.Parameter(default="") + workspace = luigi.Parameter(default="artifacts") + + def requires(self): + return ImageReader(image_file=self.image_file, mask_file=self.mask_file, workspace=self.workspace) + + def output(self): + out_dir = ensure_dir(Path(self.workspace) / "pipeline") + stem = Path(self.image_file).stem + return luigi.LocalTarget(str(out_dir / f"fusion_{stem}.json")) + + def run(self): + # If we had the second modality, we would read here and stack the channels. + # For now, we're passing that single image along with the metadata. + payload = {"status": "fused", "modalities": 1, "image": str(self.image_file)} + with self.output().open("w") as f: + json.dump(payload, f, indent=2) + + +class ImageConversion(luigi.Task): + image_file = luigi.Parameter() + mask_file = luigi.Parameter(default="") + workspace = luigi.Parameter(default="artifacts") + + def requires(self): + return ImageFusion(image_file=self.image_file, mask_file=self.mask_file, workspace=self.workspace) + + def output(self): + out_dir = ensure_dir(Path(self.workspace) / "pipeline") + stem = Path(self.image_file).stem + return luigi.LocalTarget(str(out_dir / f"conversion_{stem}.json")) + + def run(self): + # Convert to SimpleITK image for later steps + sitk_img = sitk.ReadImage(str(self.image_file)) + # Type conversion/normalization + payload = {"status": "converted", "pixel_type": str(sitk_img.GetPixelIDTypeAsString())} + with self.output().open("w") as f: + json.dump(payload, f, indent=2) + + +class ImageFilter(luigi.Task): + image_file = luigi.Parameter() + mask_file = luigi.Parameter(default="") + workspace = luigi.Parameter(default="artifacts") + sigma = luigi.FloatParameter(default=1.0) + + def requires(self): + return ImageConversion(image_file=self.image_file, mask_file=self.mask_file, workspace=self.workspace) + + def output(self): + out_dir = ensure_dir(Path(self.workspace) / "pipeline") + stem = Path(self.image_file).stem + return luigi.LocalTarget(str(out_dir / f"filter_{stem}.json")) + + def run(self): + img = sitk.ReadImage(str(self.image_file)) + # Gaussian smoothing for ROI preparation + filtered = sitk.DiscreteGaussian(img, variance=self.sigma ** 2) + # Store Data + payload = {"status": "filtered", "sigma": self.sigma} + with self.output().open("w") as f: + json.dump(payload, f, indent=2) + + +class MaskRegistration(luigi.Task): + image_file = luigi.Parameter() + mask_file = luigi.Parameter(default="") + workspace = luigi.Parameter(default="artifacts") + + def requires(self): + return ImageFilter(image_file=self.image_file, mask_file=self.mask_file, workspace=self.workspace) + + def output(self): + out_dir = ensure_dir(Path(self.workspace) / "pipeline") + stem = Path(self.image_file).stem + return luigi.LocalTarget(str(out_dir / f"maskreg_{stem}.json")) + + def run(self): + # If we have a mask, we register/resample to image space. + result = {"status": "mask_registered", "mask_available": bool(self.mask_file)} + if self.mask_file: + img = sitk.ReadImage(str(self.image_file)) + msk = sitk.ReadImage(str(self.mask_file)) + # Resample mask to image geometry + resampler = sitk.ResampleImageFilter() + resampler.SetReferenceImage(img) + resampler.SetInterpolator(sitk.sitkNearestNeighbor) + resampler.SetDefaultPixelValue(0) + msk_res = resampler.Execute(msk) + # Temporary storage of PySera results + tmp_dir = ensure_dir(Path(self.workspace) / "tmp") + out_mask = Path(tmp_dir) / f"regmask_{Path(self.image_file).stem}.nii.gz" + sitk.WriteImage(msk_res, str(out_mask)) + result["registered_mask_path"] = str(out_mask) + with self.output().open("w") as f: + json.dump(result, f, indent=2) diff --git a/Luigi/engine/tasks_radiomics.py b/Luigi/engine/tasks_radiomics.py new file mode 100644 index 0000000..f641f0c --- /dev/null +++ b/Luigi/engine/tasks_radiomics.py @@ -0,0 +1,89 @@ +import os +import time +import json +import luigi +from pathlib import Path +import pysera +from engine.utils import json_safe + +class FeatureExtraction(luigi.Task): + artifacts_dir = luigi.Parameter(default="artifacts") + temp_dir = luigi.Parameter(default=r"C:\Users\Omen16\AppData\Local\ViSERA\res\memory\memmap\pysera_temp") + + def requires(self): + from engine.tasks_filter import ImageFilter + from engine.tasks_maskreg import MaskRegistration + return { + "filter": ImageFilter(artifacts_dir=self.artifacts_dir), + "maskreg": MaskRegistration(artifacts_dir=self.artifacts_dir) + } + + def output(self): + return luigi.LocalTarget(os.path.join(self.artifacts_dir, "radiomics_index.json")) + + def run(self): + # + filt_index = os.path.join(self.artifacts_dir, "filtered_index.txt") + mask_index = os.path.join(self.artifacts_dir, "mask_registered_index.txt") + with open(filt_index, "r") as f: + filtered_paths = [line.strip() for line in f if line.strip()] + with open(mask_index, "r") as f: + mask_paths = [line.strip() for line in f if line.strip()] + + results = [] + for img, mask in zip(filtered_paths, mask_paths): + start = time.time() + result = pysera.process_batch( + image_input=img, + mask_input=mask, + output_path=self.artifacts_dir, + categories="diag,morph,glcm,glrlm,glszm,ngtdm,ngldm", + dimensions="1st,3D", + bin_size=25, + roi_num=2, + roi_selection_mode="per_region", + apply_preprocessing=True, + feature_value_mode="REAL_VALUE", + min_roi_volume=50, + enable_parallelism=True, + num_workers="4", + report="info", + temporary_files_path=str(self.temp_dir), + IBSI_based_parameters={ + "radiomics_DataType": "CT", + "radiomics_DiscType": "FBS", + "radiomics_isScale": 0, + "radiomics_VoxInterp": "Nearest", + "radiomics_ROIInterp": "Nearest", + "radiomics_isotVoxSize": 2.0, + "radiomics_isotVoxSize2D": 2.0, + "radiomics_isIsot2D": 0, + "radiomics_isGLround": 0, + "radiomics_isReSegRng": 0, + "radiomics_isOutliers": 0, + "radiomics_isQuntzStat": 1, + "radiomics_ReSegIntrvl01": -1000, + "radiomics_ReSegIntrvl02": 400, + "radiomics_ROI_PV": 0.5, + "radiomics_qntz": "Uniform", + "radiomics_IVH_Type": 3, + "radiomics_IVH_DiscCont": 1, + "radiomics_IVH_binSize": 2.0, + }, + ) + elapsed = round(time.time() - start, 2) + + safe_result = json_safe(result) + case_json = os.path.join(self.artifacts_dir, f"{Path(img).name}_radiomics.json") + with open(case_json, "w", encoding="utf-8") as f: + json.dump(safe_result, f, indent=2, ensure_ascii=False) + + results.append({ + "image": img, + "mask": mask, + "elapsed_seconds": elapsed, + "result_file": case_json, + }) + + with self.output().open("w") as f: + json.dump({"radiomics_results": results}, f, indent=2, ensure_ascii=False) diff --git a/Luigi/engine/tasks_reader.py b/Luigi/engine/tasks_reader.py new file mode 100644 index 0000000..b8a82b9 --- /dev/null +++ b/Luigi/engine/tasks_reader.py @@ -0,0 +1,28 @@ +import os +import luigi +import SimpleITK as sitk +from pathlib import Path +from engine.utils import ensure_dir + +class ImageReader(luigi.Task): + data_dir = luigi.Parameter(default=os.path.join("data", "images")) + artifacts_dir = luigi.Parameter(default="artifacts") + + def output(self): + out_dir = ensure_dir(Path(self.artifacts_dir) / "reader") + return luigi.LocalTarget(str(out_dir / "reader_index.txt")) + + def run(self): + reader_dir = ensure_dir(Path(self.artifacts_dir) / "reader") + files = [os.path.join(self.data_dir, f) for f in os.listdir(self.data_dir) if f.endswith(".nii.gz")] + if not files: + raise FileNotFoundError("No images found in data/images") + converted_paths = [] + for path in files: + img = sitk.ReadImage(path) + img_float = sitk.Cast(img, sitk.sitkFloat32) + out_path = reader_dir / f"reader_{Path(path).name}" + sitk.WriteImage(img_float, str(out_path)) + converted_paths.append(str(out_path)) + with self.output().open("w") as f: + f.write("\n".join(converted_paths)) diff --git a/Luigi/engine/tasks_registration.py b/Luigi/engine/tasks_registration.py new file mode 100644 index 0000000..e645a7b --- /dev/null +++ b/Luigi/engine/tasks_registration.py @@ -0,0 +1,73 @@ +import os +import luigi +import SimpleITK as sitk +from pathlib import Path +from engine.utils import ensure_dir + +def cast_to_float32(img: sitk.Image) -> sitk.Image: + return sitk.Cast(img, sitk.sitkFloat32) + +def make_initial_transform(fixed: sitk.Image, moving: sitk.Image) -> sitk.Transform: + dim = fixed.GetDimension() + if dim == 2: + return sitk.CenteredTransformInitializer( + fixed, moving, sitk.Euler2DTransform(), sitk.CenteredTransformInitializerFilter.GEOMETRY + ) + elif dim == 3: + return sitk.CenteredTransformInitializer( + fixed, moving, sitk.VersorRigid3DTransform(), sitk.CenteredTransformInitializerFilter.GEOMETRY + ) + else: + raise RuntimeError(f"Unsupported image dimension: {dim}") + +class ImageRegistration(luigi.Task): + artifacts_dir = luigi.Parameter(default="artifacts") + + def requires(self): + from engine.tasks_reader import ImageReader + return ImageReader(artifacts_dir=self.artifacts_dir) + + def output(self): + return luigi.LocalTarget(os.path.join(self.artifacts_dir, "registered_index.txt")) + + def run(self): + # Reader Path Reader + reader_index = os.path.join(self.artifacts_dir, "reader", "reader_index.txt") + with open(reader_index, "r") as f: + reader_paths = [line.strip() for line in f if line.strip()] + + fixed_raw = sitk.ReadImage(reader_paths[0]) + fixed = cast_to_float32(fixed_raw) + + R = sitk.ImageRegistrationMethod() + if fixed.GetDimension() == 2: + R.SetMetricAsMeanSquares() + R.SetInterpolator(sitk.sitkLinear) + else: + R.SetMetricAsMattesMutualInformation(numberOfHistogramBins=50) + R.SetMetricSamplingStrategy(R.RANDOM) + R.SetMetricSamplingPercentage(0.2) + R.SetInterpolator(sitk.sitkLinear) + + R.SetOptimizerAsRegularStepGradientDescent( + learningRate=2.0, minStep=1e-4, numberOfIterations=200, gradientMagnitudeTolerance=1e-8 + ) + R.SetOptimizerScalesFromPhysicalShift() + R.SetShrinkFactorsPerLevel(shrinkFactors=[4, 2, 1]) + R.SetSmoothingSigmasPerLevel(smoothingSigmas=[2, 1, 0]) + R.SmoothingSigmasAreSpecifiedInPhysicalUnitsOn() + + out_paths = [] + for img_path in reader_paths: + moving_raw = sitk.ReadImage(img_path) + moving = cast_to_float32(moving_raw) + init_tx = make_initial_transform(fixed, moving) + R.SetInitialTransform(init_tx, inPlace=False) + final_tx = R.Execute(fixed, moving) + registered = sitk.Resample(moving, fixed, final_tx, sitk.sitkLinear, 0.0, sitk.sitkFloat32) + out_path = Path(self.artifacts_dir) / f"registered_{Path(img_path).name}" + sitk.WriteImage(registered, str(out_path)) + out_paths.append(str(out_path)) + + with self.output().open("w") as f: + f.write("\n".join(out_paths)) diff --git a/Luigi/engine/tasks_writer.py b/Luigi/engine/tasks_writer.py new file mode 100644 index 0000000..65aa0a3 --- /dev/null +++ b/Luigi/engine/tasks_writer.py @@ -0,0 +1,33 @@ +import json +import os +from pathlib import Path +import shutil +import luigi +from engine.utils import ensure_dir + + +class ImageWriter(luigi.Task): + artifacts_dir = luigi.Parameter(default="artifacts") + + def requires(self): + from engine.tasks_radiomics import FeatureExtraction + return FeatureExtraction(artifacts_dir=self.artifacts_dir) + + def output(self): + return luigi.LocalTarget(os.path.join(self.artifacts_dir, "final_output.json")) + + def run(self): + # Copy Excel file generated by PySERA to artifacts/radiomics3d + excel_src = Path(self.artifacts_dir) / "Radiomics_Results.xlsx" + if excel_src.exists(): + dst_dir = ensure_dir(Path(self.artifacts_dir) / "radiomics3d") + excel_dst = dst_dir / "Radiomics_Results.xlsx" + shutil.copy2(excel_src, excel_dst) + + # Load summary JSON from FeatureExtraction + with self.requires().output().open("r") as f: + summary = json.load(f) + + # Write final summary JSON + with self.output().open("w") as f: + json.dump(summary, f, indent=2, ensure_ascii=False) diff --git a/Luigi/engine/utils.py b/Luigi/engine/utils.py new file mode 100644 index 0000000..4e6abcd --- /dev/null +++ b/Luigi/engine/utils.py @@ -0,0 +1,27 @@ +from pathlib import Path +import json +import pandas as pd + +def ensure_dir(p): + p = Path(p) + p.mkdir(parents=True, exist_ok=True) + return p + +def json_safe(obj): + from pathlib import Path + import numpy as np + import pandas as pd + + if isinstance(obj, Path): + return str(obj) + if isinstance(obj, (str, int, float, bool)) or obj is None: + return obj + if isinstance(obj, dict): + return {str(k): json_safe(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [json_safe(v) for v in obj] + if isinstance(obj, pd.DataFrame): + return obj.to_dict(orient="records") + if isinstance(obj, np.ndarray): + return obj.tolist() + return obj # fallback diff --git a/Luigi/requirements.txt b/Luigi/requirements.txt new file mode 100644 index 0000000..acb2d77 --- /dev/null +++ b/Luigi/requirements.txt @@ -0,0 +1,39 @@ +connected-components-3d==3.26.1 +dataclasses==0.6 +et_xmlfile==2.0.0 +ImageIO==2.37.2 +joblib==1.5.2 +lazy_loader==0.4 +lockfile==0.12.2 +luigi==3.6.0 +networkx==3.6.1 +nibabel==5.3.3 +numpy==2.2.6 +opencv-python==4.12.0.88 +openpyxl==3.1.5 +packaging==25.0 +pandas==2.3.3 +pillow==12.0.0 +psutil==7.1.3 +pydicom==3.0.1 +pynrrd==1.1.3 +pysera==2.1.5 +PySide6==6.10.1 +PySide6_Addons==6.10.1 +PySide6_Essentials==6.10.1 +python-daemon==3.1.2 +python-dateutil==2.9.0.post0 +pytz==2025.2 +rt-utils==1.2.7 +scikit-image==0.25.2 +scikit-learn==1.7.2 +scipy==1.16.3 +shiboken6==6.10.1 +simpleitk==2.5.3 +six==1.17.0 +tenacity==8.5.0 +threadpoolctl==3.6.0 +tifffile==2025.10.16 +tornado==6.5.2 +typing_extensions==4.15.0 +tzdata==2025.2 diff --git a/Luigi/run_pipeline.py b/Luigi/run_pipeline.py new file mode 100644 index 0000000..95fde26 --- /dev/null +++ b/Luigi/run_pipeline.py @@ -0,0 +1,14 @@ +import luigi +import time +from engine.pipeline import RadiumaPipeline + +if __name__ == "__main__": + start = time.time() + luigi.build([RadiumaPipeline(artifacts_dir="artifacts")], local_scheduler=True) + elapsed = time.time() - start + hours, rem = divmod(elapsed, 3600) + minutes, seconds = divmod(rem, 60) + centiseconds = int((seconds - int(seconds)) * 100) + print(f"[Pipeline] total execution time: {int(hours):02}:{int(minutes):02}:{int(seconds):02}.{centiseconds:02}") + print("=== Workflow completed successfully ===") + \ No newline at end of file