Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions airflow_src/dags/impl/handler_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@
ThermoRawFileMonitorWrapper,
)

from shared.db import project_settings as project_settings_store
from shared.db.interface import (
get_project_settings,
get_raw_file_by_id,
update_raw_file,
)
Expand Down Expand Up @@ -463,7 +463,7 @@ def _is_settings_configured(raw_file: RawFile) -> bool:
instrument_type = get_instrument_settings(
raw_file.instrument_id, InstrumentKeys.TYPE
)
project_settings = get_project_settings(raw_file.project_id)
project_settings = project_settings_store.find([raw_file.project_id])
resolved_settings = resolve_scoped_settings(
project_settings,
instrument_id=raw_file.instrument_id,
Expand Down
17 changes: 9 additions & 8 deletions airflow_src/dags/impl/processor_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,23 +43,24 @@
get_job_result,
start_job,
)
from mongoengine import DoesNotExist

from shared.config_params import (
ConfigParamPlaceholders,
substitute_dummy_values,
substitute_placeholders,
)
from shared.db import project_settings as project_settings_store
from shared.db import settings as settings_store
from shared.db.interface import (
add_metrics_to_raw_file,
get_project_settings,
get_raw_file_by_id,
get_settings_by_id,
update_raw_file,
)
from shared.db.models import RawFile, Settings
from shared.db.models import RawFile
from shared.display_paths import get_current_output_display_base_path
from shared.domain.errors import NotFoundError
from shared.domain.raw_files import RawFileStatus
from shared.domain.settings import Settings
from shared.keys import SoftwareTypes
from shared.path_layout import get_output_folder_rel_path, get_raw_file_rel_path
from shared.path_views import Locations, View
Expand All @@ -75,14 +76,14 @@ def resolve_settings(raw_file_id: str) -> list[str]:

instrument_type = get_instrument_settings(instrument_id, InstrumentKeys.TYPE)
try:
project_settings = get_project_settings(raw_file.project_id)
project_settings = project_settings_store.find([raw_file.project_id])
settings_list = resolve_scoped_settings(
project_settings,
instrument_id=instrument_id,
instrument_type=instrument_type,
raw_file_id=raw_file_id,
)
except DoesNotExist as e:
except NotFoundError as e:
# this should not happen as every project has a fallback setting
raise AirflowFailException(
f"Project or settings not found for '{raw_file.project_id}'."
Expand All @@ -99,7 +100,7 @@ def resolve_settings(raw_file_id: str) -> list[str]:
base_output_path = get_internal_output_path_for_raw_file(raw_file)
base_output_path.mkdir(parents=True, exist_ok=True)

return [str(s.id) for s in settings_list] # type: ignore[unresolved-attribute]
return [s.id for s in settings_list]


def prepare_job(raw_file_id: str, settings_id: str) -> dict[str, str | int | None]:
Expand All @@ -108,7 +109,7 @@ def prepare_job(raw_file_id: str, settings_id: str) -> dict[str, str | int | Non
:return: The quanting environment as a dict, to be passed on via XCom.
"""
raw_file = get_raw_file_by_id(raw_file_id)
settings = get_settings_by_id(settings_id)
settings = settings_store.get(settings_id)

try:
runner = get_runner(settings.runner_name)
Expand Down
9 changes: 7 additions & 2 deletions airflow_src/dags/impl/watcher_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,13 @@
from mongoengine import NotUniqueError
from raw_file_wrapper_factory import RawFileWrapperFactory, get_main_file_size_from_db

from shared.db import projects as project_store
from shared.db.interface import (
add_raw_file,
delete_raw_file,
get_active_project_ids,
get_raw_files_by_names,
)
from shared.domain.projects import ProjectStatus
from shared.domain.raw_files import InstrumentFileStatus, RawFileStatus


Expand Down Expand Up @@ -217,7 +218,11 @@ def decide_raw_file_handling(ti: TaskInstance, **kwargs) -> None:
f"{len(raw_file_names_to_process)} raw files to be checked on project id: {raw_file_names_to_process}"
)

all_project_ids = get_active_project_ids()
all_project_ids = [
project.id
for project in project_store.find()
if project.status == ProjectStatus.ACTIVE
]

raw_file_names_with_decisions: dict[str, tuple[str, bool, bool]] = {}
for raw_file_name, is_collision in raw_file_names_to_process.items():
Expand Down
3 changes: 2 additions & 1 deletion airflow_src/plugins/common/paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

from pathlib import Path

from shared.db.models import RawFile, Settings
from shared.db.models import RawFile
from shared.domain.settings import Settings
from shared.path_layout import get_output_folder_rel_path
from shared.path_views import AIRFLOW_CONTAINER_VIEW, Locations

Expand Down
8 changes: 4 additions & 4 deletions airflow_src/tests/dags/impl/test_handler_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -1455,7 +1455,7 @@ def test_start_acquisition_processor_skipped_if_not_suitable(


@patch("dags.impl.handler_impl.resolve_scoped_settings", return_value=[MagicMock()])
@patch("dags.impl.handler_impl.get_project_settings", return_value=[MagicMock()])
@patch("shared.db.project_settings.find", return_value=[MagicMock()])
@patch("dags.impl.handler_impl.get_instrument_settings", return_value="thermo")
def test_is_settings_configured_returns_true_when_settings_exist(
mock_get_instrument_settings: MagicMock, # noqa: ARG001
Expand All @@ -1473,12 +1473,12 @@ def test_is_settings_configured_returns_true_when_settings_exist(

# then
assert result is True
mock_get_settings.assert_called_once_with("project1")
mock_get_settings.assert_called_once_with(["project1"])
mock_resolve_scoped.assert_called_once()


@patch("dags.impl.handler_impl.resolve_scoped_settings", return_value=[])
@patch("dags.impl.handler_impl.get_project_settings", return_value=[])
@patch("shared.db.project_settings.find", return_value=[])
@patch("dags.impl.handler_impl.get_instrument_settings", return_value="thermo")
def test_is_settings_configured_returns_false_when_settings_do_not_exist(
mock_get_instrument_settings: MagicMock, # noqa: ARG001
Expand All @@ -1496,5 +1496,5 @@ def test_is_settings_configured_returns_false_when_settings_do_not_exist(

# then
assert result is False
mock_get_settings.assert_called_once_with("project1")
mock_get_settings.assert_called_once_with(["project1"])
mock_resolve_scoped.assert_called_once()
22 changes: 11 additions & 11 deletions airflow_src/tests/dags/impl/test_processor_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
resolve_settings,
submit_job,
)
from mongoengine import DoesNotExist
from plugins.common.keys import (
JobStates,
Tasks,
Expand All @@ -40,6 +39,7 @@

from airflow_src.tests.helpers import container_locations, runner_view
from shared.db.models import RawFile
from shared.domain.errors import NotFoundError
from shared.domain.raw_files import RawFileStatus
from shared.keys import JobEngines
from shared.path_views import Locations, View
Expand Down Expand Up @@ -207,7 +207,7 @@ def test_create_quanting_env_custom_software(

@patch.dict(_INSTRUMENTS, {"instrument1": {"type": "thermo"}})
@patch("dags.impl.processor_impl.get_raw_file_by_id")
@patch("dags.impl.processor_impl.get_project_settings")
@patch("shared.db.project_settings.find")
@patch("dags.impl.processor_impl.resolve_scoped_settings")
@patch("dags.impl.processor_impl.get_internal_output_path_for_raw_file")
def test_resolve_settings(
Expand Down Expand Up @@ -238,7 +238,7 @@ def test_resolve_settings(
result = resolve_settings(raw_file_id="test_file.raw")

mock_get_raw_file_by_id.assert_called_once_with("test_file.raw")
mock_get_settings.assert_called_once_with("some_project_id")
mock_get_settings.assert_called_once_with(["some_project_id"])
mock_resolve_scoped.assert_called_once()
mock_get_internal_output_path.assert_called_once_with(mock_raw_file)
mock_get_internal_output_path.return_value.mkdir.assert_called_once_with(
Expand All @@ -248,7 +248,7 @@ def test_resolve_settings(


@patch("dags.impl.processor_impl.get_raw_file_by_id")
@patch("dags.impl.processor_impl.get_project_settings")
@patch("shared.db.project_settings.find")
@patch("dags.impl.processor_impl.get_instrument_settings")
def test_resolve_settings_no_project_raise(
mock_get_instrument_settings: MagicMock,
Expand All @@ -266,14 +266,14 @@ def test_resolve_settings_no_project_raise(

mock_get_instrument_settings.return_value = "thermo"

mock_get_settings.side_effect = DoesNotExist
mock_get_settings.side_effect = NotFoundError("gone")

with pytest.raises(AirflowFailException):
resolve_settings(raw_file_id="test_file.raw")


@patch("dags.impl.processor_impl.get_raw_file_by_id")
@patch("dags.impl.processor_impl.get_project_settings")
@patch("shared.db.project_settings.find")
@patch("dags.impl.processor_impl.resolve_scoped_settings")
@patch("dags.impl.processor_impl.get_instrument_settings")
def test_resolve_settings_no_settings_raise(
Expand Down Expand Up @@ -301,7 +301,7 @@ def test_resolve_settings_no_settings_raise(


@patch("dags.impl.processor_impl._create_quanting_env")
@patch("dags.impl.processor_impl.get_settings_by_id")
@patch("shared.db.settings.get")
@patch("dags.impl.processor_impl.get_raw_file_by_id")
@runner_view("slurm", backup="/some_backup_base_path")
def test_prepare_job(
Expand Down Expand Up @@ -357,7 +357,7 @@ def test_prepare_job(


@patch.dict(RUNNERS, {"win_box": _WINDOWS_RUNNER})
@patch("dags.impl.processor_impl.get_settings_by_id")
@patch("shared.db.settings.get")
@patch("dags.impl.processor_impl.get_raw_file_by_id")
def test_prepare_job_windows_runner_yields_windows_paths(
mock_get_raw_file_by_id: MagicMock,
Expand Down Expand Up @@ -552,7 +552,7 @@ def test_check_content_rejects_malicious_field(

@patch("dags.impl.processor_impl._check_content")
@patch("dags.impl.processor_impl._create_quanting_env")
@patch("dags.impl.processor_impl.get_settings_by_id")
@patch("shared.db.settings.get")
@patch("dags.impl.processor_impl.get_raw_file_by_id")
@runner_view("slurm", backup="/some_backup_base_path")
def test_prepare_job_validation_error_raises(
Expand Down Expand Up @@ -590,7 +590,7 @@ def test_prepare_job_validation_error_raises(
mock_check_content.assert_called_once_with(mock_env, mock_settings)


@patch("dags.impl.processor_impl.get_settings_by_id")
@patch("shared.db.settings.get")
@patch("dags.impl.processor_impl.get_raw_file_by_id")
def test_prepare_job_unknown_runner_raises(
mock_get_raw_file_by_id: MagicMock,
Expand Down Expand Up @@ -766,7 +766,7 @@ def test_find_next_free_run_suffix_base_not_exists(tmp_path: Path) -> None:
@patch("dags.impl.processor_impl.get_internal_output_path_for_raw_file")
@patch("dags.impl.processor_impl._check_content")
@patch("dags.impl.processor_impl._create_quanting_env")
@patch("dags.impl.processor_impl.get_settings_by_id")
@patch("shared.db.settings.get")
@patch("dags.impl.processor_impl.get_raw_file_by_id")
@runner_view("slurm", backup="/some_backup_base_path")
def test_prepare_job_add_mode( # noqa: PLR0913
Expand Down
11 changes: 9 additions & 2 deletions airflow_src/tests/dags/impl/test_watcher_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
from plugins.common.keys import OpArgs, XComKeys

from shared.db.models import RawFile
from shared.domain.projects import ProjectStatus
from shared.domain.raw_files import InstrumentFileStatus, RawFileStatus
from shared.tests.factories import project

SOME_INSTRUMENT_ID = "_test1_"

Expand Down Expand Up @@ -295,7 +297,7 @@ def test_get_unknown_raw_files_with_empty_directory(
mock_put_xcom.assert_called_once_with(ti, XComKeys.RAW_FILE_NAMES_TO_PROCESS, {})


@patch("dags.impl.watcher_impl.get_active_project_ids")
@patch("shared.db.projects.find")
@patch("dags.impl.watcher_impl.get_xcom")
@patch("dags.impl.watcher_impl.get_unique_project_id")
@patch("dags.impl.watcher_impl._file_meets_age_criterion")
Expand All @@ -310,7 +312,12 @@ def test_decide_raw_file_handling(
"""A test for the decide_raw_file_handling function."""
mock_ti = MagicMock()
mock_get_xcom.return_value = {"file1": "", "file2": "", "file3": "123-"}
mock_get_project_ids.return_value = ["project1", "project2"]
# archived projects do not take new raw files
mock_get_project_ids.return_value = [
project(id="project1"),
project(id="project2"),
project(id="archived", status=ProjectStatus.INACTIVE),
]
mock_get_unique.side_effect = [None, "project1", "project2"]
mock_file_meets.side_effect = [True, True, False]

Expand Down
1 change: 0 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ ignore = [
# legacy, cf. design_docs/DB_INTERFACE_PLAN.md: remove an entry once the module is ported
"shared/path_layout.py" = ["TID251"]
"shared/display_paths.py" = ["TID251"]
"shared/settings_scope_resolver.py" = ["TID251"]
"shared/tests/test_path_layout.py" = ["TID251"]
"shared/tests/test_display_paths.py" = ["TID251"]
"shared/tests/test_runners.py" = ["TID251"]
Expand Down
12 changes: 8 additions & 4 deletions shared/db/_mongo/project_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from shared.db._mongo.errors import translate_errors
from shared.db._mongo.mappers import to_project_settings, to_settings
from shared.db._mongo.settings import settings_by_unique_name
from shared.domain.errors import NotFoundError
from shared.domain.projects import ProjectSettings


Expand All @@ -20,10 +21,13 @@ def find(project_ids: list[str] | None) -> list[ProjectSettings]:
settings_doc.pk: to_settings(settings_doc)
for settings_doc in models.Settings.objects(id__in=settings_ids)
}
return [
to_project_settings(doc, settings_by_id[doc.to_mongo()["settings"]])
for doc in docs
]
result = []
for doc in docs:
settings_id = doc.to_mongo()["settings"]
if settings_id not in settings_by_id:
raise NotFoundError(f"Settings of assignment {doc.pk} do not exist")
result.append(to_project_settings(doc, settings_by_id[settings_id]))
return result


@translate_errors
Expand Down
28 changes: 0 additions & 28 deletions shared/db/interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,10 @@
from shared.db.engine import connect_db
from shared.db.models import (
Metrics,
Project,
ProjectSettings,
RawFile,
Settings,
SlurmJob,
User,
get_slurm_job_db_id,
)
from shared.domain.projects import ProjectStatus
from shared.domain.raw_files import FileInfoItem, InstrumentFileStatus
from shared.domain.slurm_jobs import SlurmJobQueueStates

Expand Down Expand Up @@ -195,29 +190,6 @@ def add_metrics_to_raw_file( # noqa: PLR0913
).save()


def get_active_project_ids() -> list[str]:
"""Get the ids of all projects that are neither archived nor deleted."""
connect_db()
return [p.id for p in Project.objects(status=ProjectStatus.ACTIVE)]


def get_project_settings(project_id: str) -> list[ProjectSettings]:
"""Get all project-settings assignments for a project."""
connect_db()
return list(ProjectSettings.objects(project=project_id).order_by("created_at_"))


def _get_owner(owner_email: str | None) -> User | None:
"""Get the user to set as owner, raises if it does not exist."""
return User.objects.get(email=owner_email) if owner_email else None


def get_settings_by_id(settings_id: str) -> Settings:
"""Get a settings entry by its ID."""
connect_db()
return Settings.objects.get(id=settings_id)


def update_slurm_jobs(cluster_id: str, states: dict[str, str]) -> None:
"""Store the `states` (job id -> state) of all jobs in the queue of the cluster `cluster_id`.

Expand Down
3 changes: 2 additions & 1 deletion shared/path_layout.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@

from pathlib import Path

from shared.db.models import RawFile, Settings, get_created_at_year_month
from shared.db.models import RawFile, get_created_at_year_month
from shared.domain import raw_files
from shared.domain.settings import Settings

OUTPUT_FOLDER_PREFIX = "out_"

Expand Down
Loading
Loading