From e86b4169c9cb116341e72154bed2456dcc207558 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Fri, 17 Jul 2026 12:19:18 +0200 Subject: [PATCH 1/4] indexed fields implementation --- AGENTS.md | 13 + dbzero/dbzero/dbzero.pyi | 36 +- dbzero/dbzero/memo.py | 40 +- python_tests/test_fields_of.py | 99 +++ python_tests/test_indexed_fields.py | 806 ++++++++++++++++++ src/dbzero/bindings/python/Memo.cpp | 43 +- .../bindings/python/MemoTypeDecoration.cpp | 10 +- .../bindings/python/MemoTypeDecoration.hpp | 5 +- src/dbzero/bindings/python/PyAPI.cpp | 152 +++- src/dbzero/bindings/python/PyAPI.hpp | 2 + src/dbzero/bindings/python/PyFieldRef.cpp | 428 ++++++++++ src/dbzero/bindings/python/PyFieldRef.hpp | 34 + src/dbzero/bindings/python/PyInternalAPI.cpp | 159 ++-- src/dbzero/bindings/python/PyInternalAPI.hpp | 3 + src/dbzero/bindings/python/PyToolkit.cpp | 78 +- src/dbzero/bindings/python/PyToolkit.hpp | 12 +- .../bindings/python/collections/PyIndex.cpp | 94 +- .../bindings/python/collections/PyIndex.hpp | 9 +- src/dbzero/bindings/python/dbzero.cpp | 8 +- .../bindings/python/iter/PyObjectIterable.cpp | 4 + .../bindings/python/types/PyObjectId.hpp | 2 +- .../core/collections/range_tree/IndexBase.cpp | 40 +- .../core/collections/range_tree/IndexBase.hpp | 27 +- src/dbzero/object_model/Utils.hpp | 18 + src/dbzero/object_model/class/Class.cpp | 389 ++++++++- src/dbzero/object_model/class/Class.hpp | 81 +- .../object_model/class/ClassFactory.cpp | 199 ++++- src/dbzero/object_model/index/Index.cpp | 160 +++- src/dbzero/object_model/index/Index.hpp | 44 +- .../object_model/index/IndexBuilder.hpp | 49 +- src/dbzero/object_model/object/Object.cpp | 48 ++ .../object_model/object/ObjectImplBase.cpp | 51 +- .../object_model/object/ObjectImplBase.hpp | 7 +- .../object_model/object/ObjectInitializer.cpp | 40 + .../object_model/object/ObjectInitializer.hpp | 11 + .../object_model/tags/ObjectIterable.cpp | 24 +- .../object_model/tags/ObjectIterable.hpp | 33 +- src/dbzero/object_model/tags/TagIndex.cpp | 7 +- 38 files changed, 3052 insertions(+), 213 deletions(-) create mode 100644 python_tests/test_fields_of.py create mode 100644 python_tests/test_indexed_fields.py create mode 100644 src/dbzero/bindings/python/PyFieldRef.cpp create mode 100644 src/dbzero/bindings/python/PyFieldRef.hpp create mode 100644 src/dbzero/object_model/Utils.hpp diff --git a/AGENTS.md b/AGENTS.md index 1809986d..2e159961 100755 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,3 +79,16 @@ Consequences for any code that mutates a `MorphingBIndex`: - Destructive shortcuts (destroying and rebuilding the whole bindex, or erasing it entirely from its parent) avoid the issue since no stale reference remains. When adding a new mutating path that operates on a `MorphingBIndex`, treat re-syncing any externally held `{address, type}` as mandatory, not an optimization. Collection-specific handling (where these pairs live, which paths must re-sync) is documented at the top of the relevant `.cpp` files. + +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +When the user types `/graphify`, use the installed graphify skill or instructions before doing anything else. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/dbzero/dbzero/dbzero.pyi b/dbzero/dbzero/dbzero.pyi index 6b0c871c..c4a18988 100755 --- a/dbzero/dbzero/dbzero.pyi +++ b/dbzero/dbzero/dbzero.pyi @@ -2,7 +2,7 @@ Type stubs for dbzero module. """ -from typing import Any, Optional, Iterable, Dict, List, Tuple, Union, Callable, Sequence +from typing import Any, Optional, Iterable, Dict, List, Tuple, Union, Callable, Sequence, overload from .interfaces import ( Memo, MemoWeakProxy, QueryObject, Tag, TagSet, EnumValue, ListObject, IndexObject, TupleObject, SetObject, DictObject, ByteArrayObject, @@ -11,6 +11,28 @@ from .interfaces import ( # Core workspace management functions +class FieldRef: + ... + +class FieldNamespace: + def __getattr__(self, field_name: str) -> FieldRef: + ... + + def __getitem__(self, field_name: str) -> FieldRef: + ... + +def tag_fields(*field_names: str) -> Callable[[type], type]: + """Declare memo fields that should later be tag-backed.""" + ... + +def indexed_fields(*field_names: str) -> Callable[[type], type]: + """Declare memo fields that should be backed by managed indexes.""" + ... + +def fields_of(memo_type: type) -> FieldNamespace: + """Return a namespace for first-class memo field references.""" + ... + def open(prefix_name: str, open_mode: str = "rw", **kwargs: Any) -> None: """Open a data prefix and set it as the current working context. @@ -780,6 +802,18 @@ def index() -> IndexObject: """ ... +@overload +def index_of(memo_type: type, field_name: str, *, prefix: Optional[str] = None) -> IndexObject: + ... + +@overload +def index_of(field: FieldRef, *, prefix: Optional[str] = None) -> IndexObject: + ... + +def index_of(*args: Any, prefix: Optional[str] = None) -> IndexObject: + """Return the managed query index for an indexed memo field.""" + ... + def tuple(iterable: Iterable[Any] = (), /) -> TupleObject: """Create a persistent, immutable sequence object. diff --git a/dbzero/dbzero/memo.py b/dbzero/dbzero/memo.py index afbeab3c..641a14f7 100755 --- a/dbzero/dbzero/memo.py +++ b/dbzero/dbzero/memo.py @@ -8,43 +8,56 @@ __DBZERO_TAG_FIELDS_ATTR = "__DBZERO_TAG_FIELDS_ATTR" +__DBZERO_INDEXED_FIELDS_ATTR = "__DBZERO_INDEXED_FIELDS_ATTR" -def _normalize_tag_fields(field_names): - tag_fields = [] +def _normalize_field_names(field_names, decorator_name): + fields = [] seen = set() for field_name in field_names: if not isinstance(field_name, str): - raise TypeError("tag_fields arguments must be strings") + raise TypeError(f"{decorator_name} arguments must be strings") if field_name not in seen: seen.add(field_name) - tag_fields.append(field_name) - return tuple(tag_fields) + fields.append(field_name) + return tuple(fields) -def _merge_tag_field_declarations(*declarations): - tag_fields = [] +def _merge_field_declarations(*declarations): + fields = [] seen = set() for declaration in declarations: for field_name in declaration: if field_name not in seen: seen.add(field_name) - tag_fields.append(field_name) - return tuple(tag_fields) + fields.append(field_name) + return tuple(fields) def tag_fields(*field_names): - new_fields = _normalize_tag_fields(field_names) + new_fields = _normalize_field_names(field_names, "tag_fields") def wrap(cls): existing_fields = getattr(cls, __DBZERO_TAG_FIELDS_ATTR, ()) - merged_fields = _merge_tag_field_declarations(existing_fields, new_fields) + merged_fields = _merge_field_declarations(existing_fields, new_fields) setattr(cls, __DBZERO_TAG_FIELDS_ATTR, merged_fields) return cls return wrap +def indexed_fields(*field_names): + new_fields = _normalize_field_names(field_names, "indexed_fields") + + def wrap(cls): + existing_fields = cls.__dict__.get(__DBZERO_INDEXED_FIELDS_ATTR, ()) + merged_fields = _merge_field_declarations(existing_fields, new_fields) + setattr(cls, __DBZERO_INDEXED_FIELDS_ATTR, merged_fields) + return cls + + return wrap + + def migration(func: Callable) -> Callable: """Decorator for marking a function as a migration function""" func._db0_migration = None @@ -310,11 +323,14 @@ def wrap(cls_): init_vars = [] tag_field_names = list(getattr(cls_, __DBZERO_TAG_FIELDS_ATTR, ())) + indexed_field_names = list(cls_.__dict__.get(__DBZERO_INDEXED_FIELDS_ATTR, ())) wrapped = _wrap_memo_type(cls_, py_file = getfile(cls_), py_init_vars = init_vars, py_dyn_prefix = dyn_prefix, \ - py_migrations = list(find_migrations(cls_)) if is_singleton else None, py_tag_fields = tag_field_names, **kwargs + py_migrations = list(find_migrations(cls_)) if is_singleton else None, py_tag_fields = tag_field_names, + py_indexed_fields = indexed_field_names, **kwargs ) setattr(wrapped, __DBZERO_TAG_FIELDS_ATTR, tuple(tag_field_names)) + setattr(wrapped, __DBZERO_INDEXED_FIELDS_ATTR, tuple(indexed_field_names)) # Call __init_subclass__ on the wrapped class for any base that defines it. # Python normally calls __init_subclass__ before the decorator runs, so the diff --git a/python_tests/test_fields_of.py b/python_tests/test_fields_of.py new file mode 100644 index 00000000..70f15a0d --- /dev/null +++ b/python_tests/test_fields_of.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# Copyright (c) 2026 DBZero Software sp. z o.o. + +from dataclasses import dataclass +import pickle + +import pytest +import dbzero as db0 + + +def test_fields_of_rejects_non_memo_type(): + with pytest.raises(TypeError, match="memo type"): + db0.fields_of(object) + + +def test_fields_of_preserves_dataclass_class_attributes(db0_fixture): + @db0.memo + @dataclass + class FieldRefDataclassTask: + title: str + priority: int = 0 + + assert FieldRefDataclassTask.priority == 0 + assert not hasattr(FieldRefDataclassTask, "__fields__") + + ns = db0.fields_of(FieldRefDataclassTask) + assert ns is db0.fields_of(FieldRefDataclassTask) + assert ns.priority is ns.priority + assert repr(ns.priority).endswith(".FieldRefDataclassTask.priority>") + assert "priority" in dir(ns) + + with pytest.raises(AttributeError): + ns.missing + + +def test_fields_of_inherits_declared_field_identity(db0_fixture): + @db0.memo + class FieldRefBase: + def __init__(self, created_at): + self.created_at = created_at + + @db0.memo + class FieldRefDerived(FieldRefBase): + def __init__(self, created_at, priority): + super().__init__(created_at) + self.priority = priority + + assert db0.fields_of(FieldRefDerived).created_at is db0.fields_of(FieldRefBase).created_at + assert db0.fields_of(FieldRefDerived).priority is db0.fields_of(FieldRefDerived).priority + + +def test_fields_of_supports_explicit_dynamic_names(db0_fixture): + @db0.memo + class FieldRefDynamic: + pass + + dynamic = db0.fields_of(FieldRefDynamic)["from"] + assert dynamic == db0.fields_of(FieldRefDynamic)["from"] + assert dynamic is db0.fields_of(FieldRefDynamic)["from"] + + with pytest.raises(TypeError): + db0.fields_of(FieldRefDynamic)[123] + + with pytest.raises(Exception, match="Invalid persistent field name"): + db0.fields_of(FieldRefDynamic)["_X__hidden"] + + +def test_field_ref_and_namespace_are_not_picklable(db0_fixture): + @db0.memo + class FieldRefPickle: + def __init__(self, value): + self.value = value + + ns = db0.fields_of(FieldRefPickle) + with pytest.raises(TypeError): + pickle.dumps(ns) + with pytest.raises(TypeError): + pickle.dumps(ns.value) + + +def test_index_of_accepts_field_ref(db0_fixture): + @db0.memo + @db0.indexed_fields("priority") + class FieldRefIndexedTask: + def __init__(self, name): + self.name = name + + by_string = db0.index_of(FieldRefIndexedTask, "priority") + by_ref = db0.index_of(db0.fields_of(FieldRefIndexedTask).priority) + low = FieldRefIndexedTask("low") + high = FieldRefIndexedTask("high") + low.priority = 1 + high.priority = 5 + + assert [item.name for item in db0.find(FieldRefIndexedTask, by_string.select(1, 1))] == ["low"] + assert [item.name for item in db0.find(FieldRefIndexedTask, by_ref.select(5, 5))] == ["high"] + + with pytest.raises(Exception, match="does not accept a field_name"): + db0.index_of(db0.fields_of(FieldRefIndexedTask).priority, "priority") diff --git a/python_tests/test_indexed_fields.py b/python_tests/test_indexed_fields.py new file mode 100644 index 00000000..627941e4 --- /dev/null +++ b/python_tests/test_indexed_fields.py @@ -0,0 +1,806 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# Copyright (c) 2026 DBZero Software sp. z o.o. + +import pytest +import dbzero as db0 +from dbzero.dbzero import _get_indexed_fields +from .conftest import DB0_DIR + + +_INDEXED_FIELDS_ATTR = "__DBZERO_INDEXED_FIELDS_ATTR" + + +def _indexed_names(memo_type, index, min_key=None, max_key=None, **kwargs): + return [item.name for item in db0.find(memo_type, index.select(min_key, max_key, **kwargs))] + + +def _indexed_uuid_set(memo_type, index, min_key=None, max_key=None, **kwargs): + return {db0.uuid(item) for item in db0.find(memo_type, index.select(min_key, max_key, **kwargs))} + + +def test_indexed_fields_are_empty_before_class_is_materialized(db0_fixture): + @db0.memo + @db0.indexed_fields("date", "priority") + class IndexedFieldsUnmaterialized: + def __init__(self, date, priority=None): + self.date = date + self.priority = priority + + assert _get_indexed_fields(IndexedFieldsUnmaterialized) == () + + +def test_indexed_fields_report_materialized_declared_fields_only(db0_fixture): + @db0.indexed_fields("date", "priority") + @db0.memo + class IndexedFieldsMaterialized: + def __init__(self, date, priority=None): + self.date = date + + IndexedFieldsMaterialized(20260715) + + assert _get_indexed_fields(IndexedFieldsMaterialized) == ("date",) + + +def test_multiple_indexed_fields_decorators_merge_in_order(db0_fixture): + @db0.memo + @db0.indexed_fields("priority") + @db0.indexed_fields("date") + class IndexedFieldsMergedBeforeMemo: + def __init__(self, date, priority): + self.date = date + self.priority = priority + + IndexedFieldsMergedBeforeMemo(20260715, 1) + assert _get_indexed_fields(IndexedFieldsMergedBeforeMemo) == ("date", "priority") + + @db0.indexed_fields("owner") + @db0.memo + @db0.indexed_fields("date") + class IndexedFieldsMergedAroundMemo: + def __init__(self, date, owner): + self.date = date + self.owner = owner + + IndexedFieldsMergedAroundMemo(20260715, 7) + assert _get_indexed_fields(IndexedFieldsMergedAroundMemo) == ("date", "owner") + + +def test_duplicate_indexed_fields_are_ignored(db0_fixture): + @db0.memo + @db0.indexed_fields("priority", "date", "priority") + @db0.indexed_fields("date", "owner") + class IndexedFieldsDeduplicated: + def __init__(self, date, owner, priority): + self.date = date + self.owner = owner + self.priority = priority + + IndexedFieldsDeduplicated(20260715, 7, 1) + assert _get_indexed_fields(IndexedFieldsDeduplicated) == ("date", "owner", "priority") + + +def test_indexed_fields_requires_string_names(): + with pytest.raises(TypeError): + db0.indexed_fields("date", 123) + + +def test_empty_indexed_fields_declaration(db0_fixture): + @db0.memo + @db0.indexed_fields() + class IndexedFieldsEmpty: + pass + + assert _get_indexed_fields(IndexedFieldsEmpty) == () + + +def test_existing_fields_are_resolved_during_type_attachment(db0_fixture): + @db0.memo + class ExistingIndexedFields: + def __init__(self, date): + self.date = date + + ExistingIndexedFields(20260715) + + @db0.indexed_fields("date", "missing") + @db0.memo + class ExistingIndexedFields: + def __init__(self, date): + self.date = date + + assert _get_indexed_fields(ExistingIndexedFields) == ("date",) + + +def test_indexed_fields_on_plain_class_only_stores_python_metadata(): + @db0.indexed_fields("date") + class PlainIndexedFieldsClass: + pass + + assert not db0.is_memo(PlainIndexedFieldsClass) + assert getattr(PlainIndexedFieldsClass, _INDEXED_FIELDS_ATTR) == ("date",) + + with pytest.raises(TypeError): + _get_indexed_fields(PlainIndexedFieldsClass) + + +def test_index_of_validates_arguments_and_materialization(db0_fixture): + @db0.memo + @db0.indexed_fields("date") + class IndexedFieldsValidationUnmaterialized: + def __init__(self, date): + self.date = date + + with pytest.raises(Exception, match="memo type"): + db0.index_of(object, "date") + + with pytest.raises(Exception, match="field_name must be a string"): + db0.index_of(IndexedFieldsValidationUnmaterialized, 123) + + assert _get_indexed_fields(IndexedFieldsValidationUnmaterialized) == () + empty_index = db0.index_of(IndexedFieldsValidationUnmaterialized, "date") + assert len(empty_index) == 0 + assert _get_indexed_fields(IndexedFieldsValidationUnmaterialized) == ("date",) + + @db0.indexed_fields("date") + @db0.memo + class IndexedFieldsValidation: + def __init__(self, date): + self.date = date + self.owner = "alice" + + IndexedFieldsValidation(20260715) + + with pytest.raises(Exception, match="Unknown field"): + db0.index_of(IndexedFieldsValidation, "missing") + + with pytest.raises(Exception, match="not an indexed field"): + db0.index_of(IndexedFieldsValidation, "owner") + + +def test_index_of_accepts_declared_dynamic_indexed_field_before_assignment(db0_fixture): + @db0.indexed_fields("later") + @db0.memo + class IndexedFieldsDynamic: + pass + + item = IndexedFieldsDynamic() + empty_index = db0.index_of(IndexedFieldsDynamic, "later") + assert len(empty_index) == 0 + + item.later = 7 + index = db0.index_of(IndexedFieldsDynamic, "later") + assert len(index) == 1 + assert [obj.later for obj in db0.find(IndexedFieldsDynamic, index.select(7, 7))] == [7] + + +def test_index_of_explicit_scoped_prefix_must_be_open(db0_fixture): + prefix_name = "indexed-fields-scoped-prefix" + + @db0.indexed_fields("priority") + @db0.memo(prefix=prefix_name) + class IndexedFieldsScoped: + def __init__(self, priority): + self.priority = priority + + with pytest.raises(Exception, match="Prefix is not open"): + db0.index_of(IndexedFieldsScoped, "priority", prefix=prefix_name) + + implicit_index = db0.index_of(IndexedFieldsScoped, "priority") + assert len(implicit_index) == 0 + + +def test_index_of_returns_managed_index(db0_fixture): + @db0.indexed_fields("priority") + @db0.memo + class IndexedFieldsManaged: + def __init__(self, priority): + self.priority = priority + + item = IndexedFieldsManaged(1) + index = db0.index_of(IndexedFieldsManaged, "priority") + + assert len(index) == 1 + + for operation in ( + lambda: index.add(2, item), + lambda: index.remove(1, item), + lambda: index.clear(), + lambda: index.flush(), + ): + with pytest.raises(Exception, match="managed indexes are read-only"): + operation() + + +def test_index_of_managed_index_remains_sealed_after_reopen_and_stored_reference(db0_fixture): + @db0.indexed_fields("priority") + @db0.memo(id="dbzero-software/dbzero/tests/indexed-fields-reopen-task") + class IndexedFieldsReopenTask: + def __init__(self, name, priority): + self.name = name + self.priority = priority + + @db0.memo(id="dbzero-software/dbzero/tests/indexed-fields-index-holder") + class IndexedFieldsIndexHolder: + def __init__(self, index, items): + self.index = index + self.items = items + + low = IndexedFieldsReopenTask("low", 1) + high = IndexedFieldsReopenTask("high", 5) + holder = IndexedFieldsIndexHolder(db0.index_of(IndexedFieldsReopenTask, "priority"), [low, high]) + holder_id = db0.uuid(holder) + + for operation in ( + lambda: holder.index.add(3, holder), + lambda: holder.index.clear(), + ): + with pytest.raises(Exception, match="managed indexes are read-only"): + operation() + + db0.commit() + db0.close() + db0.init(DB0_DIR) + db0.open("my-test-prefix") + + @db0.indexed_fields("priority") + @db0.memo(id="dbzero-software/dbzero/tests/indexed-fields-reopen-task") + class IndexedFieldsReopenTask: + def __init__(self, name, priority): + self.name = name + self.priority = priority + + @db0.memo(id="dbzero-software/dbzero/tests/indexed-fields-index-holder") + class IndexedFieldsIndexHolder: + def __init__(self, index, items): + self.index = index + self.items = items + + reopened_holder = db0.fetch(holder_id, IndexedFieldsIndexHolder) + stored_index = reopened_holder.index + resolved_index = db0.index_of(IndexedFieldsReopenTask, "priority") + + assert {item.name for item in db0.find(IndexedFieldsReopenTask, stored_index.select(1, 5))} == {"low", "high"} + assert {item.name for item in db0.find(IndexedFieldsReopenTask, resolved_index.select(1, 5))} == {"low", "high"} + + for operation in ( + lambda: stored_index.add(3, reopened_holder), + lambda: stored_index.remove(1, reopened_holder), + lambda: stored_index.clear(), + lambda: stored_index.flush(), + lambda: resolved_index.add(3, reopened_holder), + lambda: resolved_index.remove(1, reopened_holder), + lambda: resolved_index.clear(), + lambda: resolved_index.flush(), + ): + with pytest.raises(Exception, match="managed indexes are read-only"): + operation() + + +def test_indexed_field_rejects_unsupported_string_keys(db0_fixture): + @db0.indexed_fields("code") + @db0.memo + class IndexedFieldsUnsupportedKey: + def __init__(self, code): + self.code = code + + with pytest.raises(Exception, match="Unsupported index key type"): + IndexedFieldsUnsupportedKey("alpha") + + item = IndexedFieldsUnsupportedKey(3) + index = db0.index_of(IndexedFieldsUnsupportedKey, "code") + + assert _get_indexed_fields(IndexedFieldsUnsupportedKey) == ("code",) + assert len(index) == 1 + assert [obj.code for obj in db0.find(IndexedFieldsUnsupportedKey, index.select(3, 3))] == [3] + + with pytest.raises(Exception, match="does not allow adding key type|Unsupported index key type"): + item.code = "beta" + + assert item.code == 3 + assert len(index) == 1 + assert [obj.code for obj in db0.find(IndexedFieldsUnsupportedKey, index.select(3, 3))] == [3] + + +def test_indexed_field_index_supports_queries_sorting_and_updates(db0_fixture): + @db0.indexed_fields("priority") + @db0.memo + class IndexedFieldsTask: + def __init__(self, name, priority): + self.name = name + self.priority = priority + + low = IndexedFieldsTask("low", 1) + high = IndexedFieldsTask("high", 5) + mid = IndexedFieldsTask("mid", 3) + none_item = IndexedFieldsTask("none", None) + index = db0.index_of(IndexedFieldsTask, "priority") + + assert len(index) == 4 + assert [item.name for item in db0.find(IndexedFieldsTask, index.select(2, 5))] == ["mid", "high"] + with pytest.raises(Exception, match="Passive index queries require at least one non-passive positive predicate"): + list(index.select(1, 5)) + + assert [item.name for item in index.sort(db0.find(IndexedFieldsTask))] == ["low", "mid", "high", "none"] + assert [item.name for item in index.sort(db0.find(IndexedFieldsTask), null_first=True)] == [ + "none", "low", "mid", "high", + ] + assert [item.name for item in db0.find(IndexedFieldsTask, index.select(None, None, null_first=True))] == [ + "none", "mid", "high", "low", + ] + + mid.priority = 7 + assert [item.name for item in db0.find(IndexedFieldsTask, index.select(3, 3))] == [] + assert [item.name for item in db0.find(IndexedFieldsTask, index.select(7, 7))] == ["mid"] + + del high.priority + assert len(index) == 3 + assert [item.name for item in db0.find(IndexedFieldsTask, index.select(5, 5))] == [] + + high.priority = 2 + assert len(index) == 4 + assert [item.name for item in db0.find(IndexedFieldsTask, index.select(2, 2))] == ["high"] + + db0.delete(low) + assert len(index) == 3 + assert {item.name for item in db0.find(IndexedFieldsTask, index.select(1, 7))} == {"high", "mid"} + + +def test_index_of_resolves_inherited_indexed_fields(db0_fixture): + @db0.indexed_fields("date") + @db0.memo + class IndexedFieldsBase: + def __init__(self, date): + self.date = date + + @db0.indexed_fields("priority") + @db0.memo + class IndexedFieldsDerived(IndexedFieldsBase): + def __init__(self, date, priority): + super().__init__(date) + self.priority = priority + + base = IndexedFieldsBase(1) + derived = IndexedFieldsDerived(2, 10) + + base_index = db0.index_of(IndexedFieldsDerived, "date") + assert {item.date for item in db0.find(IndexedFieldsBase, base_index.select(1, 2))} == {base.date, derived.date} + + derived_index = db0.index_of(IndexedFieldsDerived, "priority") + assert [item.priority for item in db0.find(IndexedFieldsDerived, derived_index.select(10, 10))] == [10] + + with pytest.raises(Exception, match="Unknown field|not an indexed field"): + db0.index_of(IndexedFieldsBase, "priority") + + +def test_indexed_field_added_declaration_migrates_existing_objects_and_descendants(db0_fixture): + @db0.memo + class MigratedIndexedBase: + def __init__(self, name, priority=None, assign_priority=True): + self.name = name + if assign_priority: + self.priority = priority + + @db0.memo + class MigratedIndexedDerived(MigratedIndexedBase): + pass + + base = MigratedIndexedBase("base", 3) + derived = MigratedIndexedDerived("derived", 7) + none_item = MigratedIndexedDerived("none", None) + absent = MigratedIndexedBase("absent", assign_priority=False) + + @db0.indexed_fields("priority") + @db0.memo + class MigratedIndexedBase: + def __init__(self, name, priority=None, assign_priority=True): + self.name = name + if assign_priority: + self.priority = priority + + @db0.memo + class MigratedIndexedDerived(MigratedIndexedBase): + pass + + index = db0.index_of(MigratedIndexedBase, "priority") + assert len(index) == 3 + assert _indexed_uuid_set(MigratedIndexedBase, index, 3, 7) == {db0.uuid(base), db0.uuid(derived)} + assert [item.name for item in index.sort(db0.find(MigratedIndexedBase), null_first=True)] == [ + "none", "base", "derived", + ] + assert db0.uuid(absent) not in _indexed_uuid_set(MigratedIndexedBase, index, None, None, null_first=True) + + +def test_indexed_field_removed_declaration_destroys_old_index_and_stops_sync(db0_fixture): + @db0.indexed_fields("priority") + @db0.memo + class RemovedMigratedIndexedField: + def __init__(self, name, priority): + self.name = name + self.priority = priority + + item = RemovedMigratedIndexedField("item", 1) + old_index = db0.index_of(RemovedMigratedIndexedField, "priority") + assert _indexed_names(RemovedMigratedIndexedField, old_index, 1, 1) == ["item"] + + @db0.memo + class RemovedMigratedIndexedField: + def __init__(self, name, priority): + self.name = name + self.priority = priority + + assert _get_indexed_fields(RemovedMigratedIndexedField) == () + with pytest.raises(Exception, match="not an indexed field|Unknown field"): + db0.index_of(RemovedMigratedIndexedField, "priority") + + item.priority = 2 + with pytest.raises(Exception): + len(old_index) + + +def test_renamed_indexed_field_preserves_managed_index_identity(db0_fixture): + @db0.indexed_fields("priority") + @db0.memo + class RenamedIndexedField: + def __init__(self, name, priority): + self.name = name + self.priority = priority + + item = RenamedIndexedField("item", 1) + old_index = db0.index_of(RenamedIndexedField, "priority") + + db0.rename_field(RenamedIndexedField, "priority", "rank") + + assert _get_indexed_fields(RenamedIndexedField) == ("rank",) + assert _indexed_names(RenamedIndexedField, old_index, 1, 1) == ["item"] + new_index = db0.index_of(RenamedIndexedField, "rank") + item.rank = 5 + assert _indexed_names(RenamedIndexedField, old_index, 1, 1) == [] + assert _indexed_names(RenamedIndexedField, new_index, 5, 5) == ["item"] + + @db0.indexed_fields("rank") + @db0.memo + class RenamedIndexedField: + def __init__(self, name, rank): + self.name = name + self.rank = rank + + assert _get_indexed_fields(RenamedIndexedField) == ("rank",) + assert _indexed_names(RenamedIndexedField, db0.index_of(RenamedIndexedField, "rank"), 5, 5) == ["item"] + + +def test_failed_indexed_field_migration_rolls_back_new_indexes_and_declaration(db0_fixture): + @db0.indexed_fields("priority") + @db0.memo + class FailedMigratedIndexedField: + def __init__(self, name, priority, code): + self.name = name + self.priority = priority + self.code = code + + valid = FailedMigratedIndexedField("valid", 1, 10) + invalid = FailedMigratedIndexedField("invalid", 2, "bad-key") + active_type = FailedMigratedIndexedField + old_index = db0.index_of(active_type, "priority") + + @db0.indexed_fields("priority", "code") + @db0.memo + class FailedMigratedIndexedField: + def __init__(self, name, priority, code): + self.name = name + self.priority = priority + self.code = code + + with pytest.raises(RuntimeError, match="Unsupported index key type|does not allow adding key type"): + FailedMigratedIndexedField("new", 3, 30) + + assert _get_indexed_fields(active_type) == ("priority",) + assert set(_indexed_names(active_type, old_index, 1, 2)) == {"valid", "invalid"} + with pytest.raises(Exception, match="not an indexed field|Unknown field"): + db0.index_of(active_type, "code") + + +def test_no_auto_migrate_raises_and_keeps_previous_indexed_field_declaration(tmp_path): + db0.init(str(tmp_path), no_auto_migrate=True) + db0.open("indexed-field-no-auto-migration") + try: + @db0.indexed_fields("priority") + @db0.memo + class NoAutoMigratedIndexedField: + def __init__(self, name, priority, owner): + self.name = name + self.priority = priority + self.owner = owner + + item = NoAutoMigratedIndexedField("item", 1, 10) + + @db0.indexed_fields("owner") + @db0.memo + class NoAutoMigratedIndexedField: + def __init__(self, name, priority, owner): + self.name = name + self.priority = priority + self.owner = owner + + with pytest.raises(db0.MigrateError): + NoAutoMigratedIndexedField("new", 2, 20) + + priority_index = db0.index_of(NoAutoMigratedIndexedField, "priority") + assert _get_indexed_fields(NoAutoMigratedIndexedField) == ("priority",) + item.priority = 3 + assert _indexed_names(NoAutoMigratedIndexedField, priority_index, 3, 3) == ["item"] + with pytest.raises(Exception, match="not an indexed field|Unknown field"): + db0.index_of(NoAutoMigratedIndexedField, "owner") + finally: + db0.close() + + +def test_explicit_migrate_applies_current_indexed_field_declaration_for_derived_type(tmp_path): + db0.init(str(tmp_path), no_auto_migrate=True) + db0.open("indexed-field-explicit-derived-migration") + try: + @db0.indexed_fields("priority") + @db0.memo + class ExplicitMigratedIndexedBase: + def __init__(self, name, priority, owner): + self.name = name + self.priority = priority + self.owner = owner + + @db0.indexed_fields("score") + @db0.memo + class ExplicitMigratedIndexedDerived(ExplicitMigratedIndexedBase): + def __init__(self, name, priority, owner, score): + super().__init__(name, priority, owner) + self.score = score + + base = ExplicitMigratedIndexedBase("base", 1, 10) + derived = ExplicitMigratedIndexedDerived("derived", 2, 20, 100) + + @db0.indexed_fields("owner") + @db0.memo + class ExplicitMigratedIndexedBase: + def __init__(self, name, priority, owner): + self.name = name + self.priority = priority + self.owner = owner + + @db0.indexed_fields("priority") + @db0.memo + class ExplicitMigratedIndexedDerived(ExplicitMigratedIndexedBase): + def __init__(self, name, priority, owner, score): + super().__init__(name, priority, owner) + self.score = score + + with pytest.raises(db0.MigrateError): + ExplicitMigratedIndexedDerived("new", 3, 30, 300) + + db0.migrate(ExplicitMigratedIndexedDerived) + + base_index = db0.index_of(ExplicitMigratedIndexedBase, "owner") + derived_index = db0.index_of(ExplicitMigratedIndexedDerived, "priority") + assert _get_indexed_fields(ExplicitMigratedIndexedBase) == ("owner",) + assert _get_indexed_fields(ExplicitMigratedIndexedDerived) == ("priority",) + assert _indexed_uuid_set(ExplicitMigratedIndexedBase, base_index, 10, 20) == { + db0.uuid(base), db0.uuid(derived), + } + assert _indexed_uuid_set(ExplicitMigratedIndexedDerived, derived_index, 2, 2) == {db0.uuid(derived)} + finally: + db0.close() + + +def test_derived_indexed_field_rejects_ancestor_overlap_after_pending_materializes(db0_fixture): + @db0.indexed_fields("shared") + @db0.memo + class PendingOverlapIndexedBase: + pass + + @db0.indexed_fields("shared") + @db0.memo + class PendingOverlapIndexedDerived(PendingOverlapIndexedBase): + pass + + item = PendingOverlapIndexedDerived() + with pytest.raises(Exception, match="already indexed|ancestor|inherited"): + item.shared = 1 + + +@pytest.mark.stress_test +def test_indexed_field_migration_inheritance_persistence_stress(tmp_path): + per_type_count = 62_500 + instance_count = 4 * per_type_count + + def expected_count(predicate, groups=1): + return groups * sum(1 for seq in range(per_type_count) if predicate(seq)) + + def query_count(memo_type, *predicates): + return len(db0.find(memo_type, *predicates)) + + def append_instances(memo_type, root): + for seq in range(per_type_count): + bad_key = "migration-must-rollback" if ( + memo_type.__name__ == "IndexedFieldsStressLeaf" and seq == per_type_count - 1 + ) else seq % 17 + root.items.append( + memo_type( + seq, + seq % 11, + seq % 13, + seq % 5, + None if seq % 19 == 0 else seq % 7, + bad_key, + ) + ) + db0.commit() + + def declare_types(base_fields, derived_a_fields, derived_b_fields, leaf_fields): + @db0.indexed_fields(*base_fields) + @db0.memo(id="dbzero-software/dbzero/tests/indexed-fields-stress-base") + class IndexedFieldsStressBase: + def __init__(self, seq, bucket, score, region, phase, bad_key): + self.seq = seq + self.bucket = bucket + self.score = score + self.region = region + self.phase = phase + self.bad_key = bad_key + + @db0.indexed_fields(*derived_a_fields) + @db0.memo(id="dbzero-software/dbzero/tests/indexed-fields-stress-derived-a") + class IndexedFieldsStressDerivedA(IndexedFieldsStressBase): + pass + + @db0.indexed_fields(*derived_b_fields) + @db0.memo(id="dbzero-software/dbzero/tests/indexed-fields-stress-derived-b") + class IndexedFieldsStressDerivedB(IndexedFieldsStressBase): + pass + + @db0.indexed_fields(*leaf_fields) + @db0.memo(id="dbzero-software/dbzero/tests/indexed-fields-stress-leaf") + class IndexedFieldsStressLeaf(IndexedFieldsStressDerivedA): + pass + + return ( + IndexedFieldsStressBase, + IndexedFieldsStressDerivedA, + IndexedFieldsStressDerivedB, + IndexedFieldsStressLeaf, + ) + + def declare_root(): + @db0.memo( + singleton=True, + id="dbzero-software/dbzero/tests/indexed-fields-stress-root", + ) + class IndexedFieldsStressRoot: + def __init__(self): + self.items = [] + + return IndexedFieldsStressRoot + + db0.init(str(tmp_path), no_auto_migrate=True, autocommit=False) + db0.open("indexed-field-inheritance-stress", slab_size=1 << 30) + try: + ( + IndexedFieldsStressBase, + IndexedFieldsStressDerivedA, + IndexedFieldsStressDerivedB, + IndexedFieldsStressLeaf, + ) = declare_types(("bucket",), ("score",), ("region",), ("phase",)) + IndexedFieldsStressRoot = declare_root() + + root = IndexedFieldsStressRoot() + for memo_type in ( + IndexedFieldsStressBase, + IndexedFieldsStressDerivedA, + IndexedFieldsStressDerivedB, + IndexedFieldsStressLeaf, + ): + append_instances(memo_type, root) + + assert len(root.items) == instance_count + old_bucket_index = db0.index_of(IndexedFieldsStressBase, "bucket") + assert len(old_bucket_index) == instance_count + + # Adding bad_key must roll back after scanning the populated hierarchy: + # one leaf has a string key, which managed indexes do not support. + ( + IndexedFieldsStressBase, + IndexedFieldsStressDerivedA, + IndexedFieldsStressDerivedB, + IndexedFieldsStressLeaf, + ) = declare_types(("bucket", "bad_key"), ("score",), ("region",), ("phase",)) + + with pytest.raises(RuntimeError, match="Unsupported index key type|does not allow adding key type"): + db0.migrate(IndexedFieldsStressBase) + + assert _get_indexed_fields(IndexedFieldsStressBase) == ("bucket",) + assert len(old_bucket_index) == instance_count + assert query_count( + IndexedFieldsStressBase, + old_bucket_index.select(3, 3), + ) == expected_count(lambda seq: seq % 11 == 3, groups=4) + with pytest.raises(Exception, match="not an indexed field|Unknown field"): + db0.index_of(IndexedFieldsStressBase, "bad_key") + + ( + IndexedFieldsStressBase, + IndexedFieldsStressDerivedA, + IndexedFieldsStressDerivedB, + IndexedFieldsStressLeaf, + ) = declare_types(("region",), ("bucket",), ("score",), ("phase",)) + + db0.migrate(IndexedFieldsStressLeaf) + db0.migrate(IndexedFieldsStressDerivedB) + db0.commit() + + assert _get_indexed_fields(IndexedFieldsStressBase) == ("region",) + assert _get_indexed_fields(IndexedFieldsStressDerivedA) == ("bucket",) + assert _get_indexed_fields(IndexedFieldsStressDerivedB) == ("score",) + assert _get_indexed_fields(IndexedFieldsStressLeaf) == ("phase",) + + db0.close() + db0.init(str(tmp_path), no_auto_migrate=True, autocommit=False) + db0.open("indexed-field-inheritance-stress", slab_size=1 << 30) + + ( + IndexedFieldsStressBase, + IndexedFieldsStressDerivedA, + IndexedFieldsStressDerivedB, + IndexedFieldsStressLeaf, + ) = declare_types(("region",), ("bucket",), ("score",), ("phase",)) + IndexedFieldsStressRoot = declare_root() + + assert len(IndexedFieldsStressRoot().items) == instance_count + + region_index = db0.index_of(db0.fields_of(IndexedFieldsStressBase).region) + bucket_index = db0.index_of(IndexedFieldsStressDerivedA, "bucket") + score_index = db0.index_of(IndexedFieldsStressDerivedB, "score") + phase_index = db0.index_of(IndexedFieldsStressLeaf, "phase") + + assert len(region_index) == instance_count + assert len(bucket_index) == 2 * per_type_count + assert len(score_index) == per_type_count + assert len(phase_index) == per_type_count + + assert query_count( + IndexedFieldsStressBase, + region_index.select(2, 2), + ) == expected_count(lambda seq: seq % 5 == 2, groups=4) + assert query_count( + IndexedFieldsStressBase, + region_index.select(None, 1), + ) == expected_count(lambda seq: seq % 5 <= 1, groups=4) + assert query_count( + IndexedFieldsStressDerivedA, + bucket_index.select(3, 3), + region_index.select(2, 4), + ) == expected_count( + lambda seq: seq % 11 == 3 and 2 <= seq % 5 <= 4, + groups=2, + ) + assert query_count( + IndexedFieldsStressLeaf, + bucket_index.select(0, 10), + phase_index.select(4, 4), + ) == expected_count(lambda seq: seq % 19 != 0 and seq % 7 == 4) + + null_first = phase_index.sort(db0.find(IndexedFieldsStressLeaf), null_first=True) + first_phases = [] + for item in null_first: + first_phases.append(item.phase) + if len(first_phases) == 32: + break + assert first_phases == [None] * 32 + + descending_regions = region_index.sort( + db0.find(IndexedFieldsStressDerivedB, score_index.select(3, 8)), + desc=True, + ) + first_regions = [] + for item in descending_regions: + assert 3 <= item.score <= 8 + first_regions.append(item.region) + if len(first_regions) == 128: + break + assert first_regions == sorted(first_regions, reverse=True) + finally: + db0.close() diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index 39f60a34..b9f24feb 100755 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -538,11 +538,9 @@ namespace db0::python auto &type = self->ext().getType(); auto member_loc = type.findField(attr_name); auto member_id = std::get<0>(member_loc); + auto field_options = type.getFieldOptions(attr_name, member_id); TagIndex *tag_index = nullptr; - if (member_id - ? type.isTagField(member_id.primary().first) - : type.isDeclaredTagField(attr_name)) - { + if (field_options[db0::object_model::FieldOptions::TAG_FIELD]) { tag_index = &self->ext().getFixture()->get(); } @@ -591,7 +589,7 @@ namespace db0::python } } else { // considered as a non-mutating operation - self->ext().setPreInit(attr_name, *maybe_type_id, value, tag_index != nullptr); + self->ext().setPreInit(attr_name, *maybe_type_id, value, field_options); } return 0; } else { @@ -635,11 +633,9 @@ namespace db0::python } auto &type = self->ext().getType(); auto member_id = std::get<0>(type.findField(attr_name)); + auto field_options = type.getFieldOptions(attr_name, member_id); db0::object_model::TagIndex *tag_index = nullptr; - if (member_id - ? type.isTagField(member_id.primary().first) - : type.isDeclaredTagField(attr_name)) - { + if (field_options[db0::object_model::FieldOptions::TAG_FIELD]) { tag_index = &self->ext().getFixture()->get(); } if (tag_index && value && value != Py_None) { @@ -655,7 +651,7 @@ namespace db0::python } } // considered as a non-mutating operation - self->ext().setPreInit(attr_name, value, tag_index != nullptr); + self->ext().setPreInit(attr_name, value, field_options); } } catch (const std::exception &e) { PyErr_SetString(PyExc_AttributeError, e.what()); @@ -1019,7 +1015,8 @@ namespace db0::python PyObject *wrapPyType(PyTypeObject *base_class, bool is_singleton, bool no_default_tags, const char *prefix_name, const char *type_id, const char *file_name, std::vector &&init_vars, PyObject *py_dyn_prefix_callable, - std::vector &&migrations, bool access_control, std::vector &&tag_fields) + std::vector &&migrations, bool access_control, std::vector &&tag_fields, + std::vector &&indexed_fields) { auto py_class = Py_BORROW(base_class); auto py_module = Py_OWN(findModule(*Py_OWN(PyObject_GetAttrString((PyObject*)*py_class, "__module__")))); @@ -1060,20 +1057,14 @@ namespace db0::python type_flags, py_dyn_prefix_callable, std::move(migrations), - std::move(tag_fields) + std::move(tag_fields), + std::move(indexed_fields) ); // add to memo type registry PyToolkit::getTypeManager().addMemoType(*new_type, type_id, std::move(type_info)); // register new type with the module where the original type was located PySafeModule_AddObject(*py_module, type_name.c_str(), new_type); - // add class fields class member to access memo type information - auto py_class_fields = Py_OWN(PyClassFields_create(*new_type)); - if (PySafeDict_SetItemString((*new_type)->tp_dict, "__fields__", py_class_fields) < 0) { - PyErr_SetString(PyExc_RuntimeError, "Failed to set __fields__"); - return nullptr; - } - return (PyObject*)new_type.steal(); } @@ -1091,12 +1082,13 @@ namespace db0::python PyObject *py_migrations = nullptr; PyObject *py_access_control = nullptr; PyObject *py_tag_fields = nullptr; + PyObject *py_indexed_fields = nullptr; static const char *kwlist[] = { "input", "singleton", "no_default_tags", "prefix", "id", "py_file", "py_init_vars", - "py_dyn_prefix", "py_migrations", "access_control", "py_tag_fields", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOOOOOOO", const_cast(kwlist), &class_obj, &py_singleton, + "py_dyn_prefix", "py_migrations", "access_control", "py_tag_fields", "py_indexed_fields", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOOOOOOOO", const_cast(kwlist), &class_obj, &py_singleton, &py_no_default_tags, &py_prefix_name, &py_type_id, &py_file_name, &py_init_vars, &py_dyn_prefix, &py_migrations, - &py_access_control, &py_tag_fields)) + &py_access_control, &py_tag_fields, &py_indexed_fields)) { return NULL; } @@ -1115,6 +1107,10 @@ namespace db0::python if (PyErr_Occurred()) { return NULL; } + auto indexed_fields = extractStringList(py_indexed_fields, "py_indexed_fields"); + if (PyErr_Occurred()) { + return NULL; + } if (py_dyn_prefix == Py_None) { py_dyn_prefix = nullptr; @@ -1130,7 +1126,8 @@ namespace db0::python auto migrations = extractMigrations(py_migrations); return wrapPyType(castToType(class_obj), is_singleton, no_default_tags, prefix_name, type_id, file_name, - std::move(init_vars), py_dyn_prefix, std::move(migrations), access_control, std::move(tag_fields) + std::move(init_vars), py_dyn_prefix, std::move(migrations), access_control, std::move(tag_fields), + std::move(indexed_fields) ); } diff --git a/src/dbzero/bindings/python/MemoTypeDecoration.cpp b/src/dbzero/bindings/python/MemoTypeDecoration.cpp index 0e10154d..72ba80e0 100755 --- a/src/dbzero/bindings/python/MemoTypeDecoration.cpp +++ b/src/dbzero/bindings/python/MemoTypeDecoration.cpp @@ -27,6 +27,7 @@ namespace db0::python , m_py_dyn_prefix_callable(other.m_py_dyn_prefix_callable) , m_migrations(std::move(other.m_migrations)) , m_tag_fields(std::move(other.m_tag_fields)) + , m_indexed_fields(std::move(other.m_indexed_fields)) { m_fixture_uuid.store(other.m_fixture_uuid.load()); other.m_fixture_uuid = 0; @@ -36,7 +37,8 @@ namespace db0::python MemoTypeDecoration::MemoTypeDecoration(shared_py_object py_module, const char *prefix_name, const char *type_id, const char *file_name, std::vector &&init_vars, MemoFlags flags, shared_py_object py_dyn_prefix_callable, - std::vector &&migrations, std::vector &&tag_fields) + std::vector &&migrations, std::vector &&tag_fields, + std::vector &&indexed_fields) : m_py_module(py_module) , m_prefix_name(prefix_name) , m_type_id(type_id) @@ -46,6 +48,7 @@ namespace db0::python , m_py_dyn_prefix_callable(py_dyn_prefix_callable) , m_migrations(std::move(migrations)) , m_tag_fields(std::move(tag_fields)) + , m_indexed_fields(std::move(indexed_fields)) { init(); } @@ -61,6 +64,7 @@ namespace db0::python m_py_dyn_prefix_callable = other.m_py_dyn_prefix_callable; m_migrations = std::move(other.m_migrations); m_tag_fields = std::move(other.m_tag_fields); + m_indexed_fields = std::move(other.m_indexed_fields); m_fixture_uuid.store(other.m_fixture_uuid.load()); other.m_fixture_uuid = 0; init(); @@ -126,6 +130,10 @@ namespace db0::python const std::vector &MemoTypeDecoration::getTagFields() const { return m_tag_fields; } + + const std::vector &MemoTypeDecoration::getIndexedFields() const { + return m_indexed_fields; + } void MemoTypeDecoration::forAllMigrations(const std::unordered_set &available_members, std::function callback) const diff --git a/src/dbzero/bindings/python/MemoTypeDecoration.hpp b/src/dbzero/bindings/python/MemoTypeDecoration.hpp index d281564c..9842fc01 100755 --- a/src/dbzero/bindings/python/MemoTypeDecoration.hpp +++ b/src/dbzero/bindings/python/MemoTypeDecoration.hpp @@ -38,7 +38,8 @@ namespace db0::python const char *file_name, std::vector &&init_vars, MemoFlags flags, shared_py_object py_dyn_prefix_callable, std::vector &&migrations, - std::vector &&tag_fields = {}); + std::vector &&tag_fields = {}, + std::vector &&indexed_fields = {}); ~MemoTypeDecoration(); @@ -69,6 +70,7 @@ namespace db0::python const std::vector &getInitVars() const; const std::vector &getTagFields() const; + const std::vector &getIndexedFields() const; // @param access_type to use for opening the prefix if UUID needs to be resolved by name // note that read-only access cannot later be upgraded to read-write @@ -111,6 +113,7 @@ namespace db0::python shared_py_object m_py_dyn_prefix_callable; std::vector m_migrations; std::vector m_tag_fields; + std::vector m_indexed_fields; // by-name migrations' index std::unordered_map m_ix_migrations; diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index 6b753f2a..17f6041c 100755 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -14,10 +14,12 @@ #include "PyReflectionAPI.hpp" #include "PyHash.hpp" #include "PyWeakProxy.hpp" +#include "PyFieldRef.hpp" #include #include #include #include +#include #include #include #include @@ -48,7 +50,6 @@ namespace db0::python { - using ObjectSharedPtr = PyTypes::ObjectSharedPtr; PyObject *tryGetCacheStats() @@ -1139,10 +1140,20 @@ namespace db0::python return runSafe(tryGetMemoClass, args[0]); } - PyObject *tryGetTagFields(PyObject *py_type) + PyObject *fieldNamesToTuple(const std::vector &names) + { + auto result = Py_OWN(PyTuple_New(names.size())); + Py_ssize_t index = 0; + for (const auto &name: names) { + PySafeTuple_SetItem(*result, index++, Py_OWN(PyUnicode_FromString(name.c_str()))); + } + return result.steal(); + } + + PyObject *tryGetClassFieldNames(PyObject *py_type, const char *api_name, bool indexed_fields) { if (!PyType_Check(py_type) || !PyAnyMemoType_Check(reinterpret_cast(py_type))) { - PyErr_SetString(PyExc_TypeError, "_get_tag_fields requires a memo type"); + PyErr_Format(PyExc_TypeError, "%s requires a memo type", api_name); return nullptr; } @@ -1155,13 +1166,7 @@ namespace db0::python return PyTuple_New(0); } - auto names = type->getTagFieldNames(); - auto result = Py_OWN(PyTuple_New(names.size())); - Py_ssize_t index = 0; - for (const auto &name: names) { - PySafeTuple_SetItem(*result, index++, Py_OWN(PyUnicode_FromString(name.c_str()))); - } - return result.steal(); + return fieldNamesToTuple(indexed_fields ? type->getIndexedFieldNames() : type->getTagFieldNames()); } PyObject *PyAPI_getTagFields(PyObject *, PyObject *const *args, Py_ssize_t nargs) @@ -1171,7 +1176,132 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "_get_tag_fields requires exactly one argument"); return nullptr; } - return runSafe(tryGetTagFields, args[0]); + return runSafe(tryGetClassFieldNames, args[0], "_get_tag_fields", false); + } + + PyObject *PyAPI_getIndexedFields(PyObject *, PyObject *const *args, Py_ssize_t nargs) + { + PY_API_FUNC + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "_get_indexed_fields requires exactly one argument"); + return nullptr; + } + return runSafe(tryGetClassFieldNames, args[0], "_get_indexed_fields", true); + } + + std::shared_ptr tryFindDeclaredIndexedField( + std::shared_ptr type, const char *field_name) + { + if (!type) { + return nullptr; + } + if (type->isDeclaredIndexedField(field_name)) { + return type; + } + return tryFindDeclaredIndexedField(type->tryGetBaseClass(), field_name); + } + + void materializeDeclaredIndexedField(std::shared_ptr type, const char *field_name) + { + if (!type || !!type->findField(field_name).first) { + return; + } + auto declaring_type = tryFindDeclaredIndexedField(type, field_name); + if (!declaring_type || !!declaring_type->findField(field_name).first) { + return; + } + auto fixture = declaring_type->getFixture(); + db0::FixtureLock lock(fixture); + declaring_type->addField(field_name, 0, declaring_type->isDeclaredTagField(field_name)); + } + + PyObject *tryIndexOf(PyObject *args, PyObject *kwargs) + { + PyObject *py_target = nullptr; + PyObject *py_field_name = nullptr; + PyObject *py_prefix = nullptr; + static const char *kwlist[] = {"memo_type", "field_name", "prefix", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O$O:index_of", const_cast(kwlist), + &py_target, &py_field_name, &py_prefix)) + { + return nullptr; + } + + shared_py_object owned_field_name; + PyObject *py_type = py_target; + if (PyFieldRef_Check(py_target)) { + if (py_field_name) { + THROWF(db0::InputException) << "index_of(FieldRef) does not accept a field_name argument"; + } + py_type = reinterpret_cast(PyFieldRef_getMemoType(py_target)); + auto field_name = PyFieldRef_getFieldName(py_target); + if (!field_name) { + THROWF(db0::InputException) << "Invalid FieldRef"; + } + owned_field_name = Py_OWN(PyUnicode_FromString(field_name)); + if (!owned_field_name) { + return nullptr; + } + py_field_name = *owned_field_name; + } else if (!py_field_name) { + THROWF(db0::InputException) << "index_of requires a memo type and field_name, or a FieldRef"; + } + + if (!PyType_Check(py_type) || !PyAnyMemoType_Check(reinterpret_cast(py_type))) { + THROWF(db0::InputException) << "First argument must be a dbzero memo type"; + } + if (!PyUnicode_Check(py_field_name)) { + THROWF(db0::InputException) << "field_name must be a string"; + } + auto field_name = PyUnicode_AsUTF8(py_field_name); + if (!field_name) { + THROWF(db0::InputException) << "Invalid field_name"; + } + + auto memo_type = reinterpret_cast(py_type); + auto prefix_name = parsePrefixName(py_prefix, "index_of"); + if (PyErr_Occurred()) { + return nullptr; + } + auto fixture = resolveMemoTypeFixture(memo_type, prefix_name); + bool can_materialize = fixture->getAccessType() == db0::AccessType::READ_WRITE; + auto &class_factory = fixture->get(); + auto type = class_factory.tryGetExistingType(memo_type); + if (!type) { + if (!can_materialize) { + if (!isValidIndexedFieldName(memo_type, field_name)) { + THROWF(db0::InputException) << "Field is not an indexed field: " << field_name; + } + return Py_OWN(IndexDefaultObject_new(fixture, true, true)).steal(); + } + type = class_factory.tryGetOrCreateType(memo_type); + if (!type) { + THROWF(db0::InputException) << "Memo type is not materialized in the resolved prefix"; + } + } + if (can_materialize) { + materializeDeclaredIndexedField(type, field_name); + } + + auto member = type->tryGetMember(field_name); + if (!member) { + if (isValidIndexedFieldName(memo_type, field_name)) { + return Py_OWN(IndexDefaultObject_new(fixture, true, true)).steal(); + } + THROWF(db0::InputException) << "Unknown field: " << field_name; + } + + auto index = type->tryGetFieldIndex(field_name); + if (!index) { + THROWF(db0::InputException) << "Field is not an indexed field: " << field_name; + } + return PyToolkit::unloadIndex(fixture, index->getAddress()).steal(); + } + + PyObject *PyAPI_indexOf(PyObject *, PyObject *args, PyObject *kwargs) + { + PY_API_FUNC + return runSafe(tryIndexOf, args, kwargs); } PyObject *tryMigrate(PyObject *py_type) diff --git a/src/dbzero/bindings/python/PyAPI.hpp b/src/dbzero/bindings/python/PyAPI.hpp index 08824b6d..24e492ef 100755 --- a/src/dbzero/bindings/python/PyAPI.hpp +++ b/src/dbzero/bindings/python/PyAPI.hpp @@ -179,6 +179,8 @@ namespace db0::python PyObject *PyAPI_getMemoClass(PyObject *, PyObject *const *args, Py_ssize_t nargs); PyObject *PyAPI_getTagFields(PyObject *, PyObject *const *args, Py_ssize_t nargs); + PyObject *PyAPI_getIndexedFields(PyObject *, PyObject *const *args, Py_ssize_t nargs); + PyObject *PyAPI_indexOf(PyObject *, PyObject *args, PyObject *kwargs); PyObject *PyAPI_migrate(PyObject *, PyObject *const *args, Py_ssize_t nargs); PyObject *PyAPI_copyPrefix(PyObject *, PyObject *args, PyObject *kwargs); diff --git a/src/dbzero/bindings/python/PyFieldRef.cpp b/src/dbzero/bindings/python/PyFieldRef.cpp new file mode 100644 index 00000000..a37a2ebf --- /dev/null +++ b/src/dbzero/bindings/python/PyFieldRef.cpp @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2026 DBZero Software sp. z o.o. + +#include "PyFieldRef.hpp" +#include "Memo.hpp" +#include "MemoTypeDecoration.hpp" +#include "PyInternalAPI.hpp" +#include "PySafeAPI.hpp" +#include +#include +#include +#include +#include + +namespace db0::python +{ + + namespace + { + constexpr const char *FIELD_NAMESPACE_ATTR = "__DBZERO_FIELD_NAMESPACE"; + + bool isDunderName(const char *name) + { + auto len = std::strlen(name); + return len >= 4 && name[0] == '_' && name[1] == '_' && name[len - 2] == '_' && name[len - 1] == '_'; + } + + void appendUnique(std::vector &names, const std::string &name) + { + if (std::find(names.begin(), names.end(), name) == names.end()) { + names.push_back(name); + } + } + + void appendDecorationNames(PyTypeObject *memo_type, std::vector &names) + { + auto &decor = MemoTypeDecoration::get(memo_type); + for (const auto &name: decor.getInitVars()) { + appendUnique(names, name); + } + for (const auto &name: decor.getTagFields()) { + appendUnique(names, name); + } + for (const auto &name: decor.getIndexedFields()) { + appendUnique(names, name); + } + } + + bool pyTupleContainsString(PyObject *tuple, const char *name) + { + if (!tuple || !PyTuple_Check(tuple)) { + return false; + } + auto size = PyTuple_Size(tuple); + for (Py_ssize_t index = 0; index < size; ++index) { + auto item = PyTuple_GetItem(tuple, index); + if (PyUnicode_Check(item)) { + auto item_name = PyUnicode_AsUTF8(item); + if (item_name && std::strcmp(item_name, name) == 0) { + return true; + } + } + } + return false; + } + + bool hasDeclaredName(PyTypeObject *memo_type, const char *name) + { + auto &decor = MemoTypeDecoration::get(memo_type); + for (const auto &field_name: decor.getInitVars()) { + if (field_name == name) { + return true; + } + } + for (const auto &field_name: decor.getTagFields()) { + if (field_name == name) { + return true; + } + } + for (const auto &field_name: decor.getIndexedFields()) { + if (field_name == name) { + return true; + } + } + + auto py_tag_fields = Py_OWN(PyObject_GetAttrString(reinterpret_cast(memo_type), "__DBZERO_TAG_FIELDS_ATTR")); + if (!py_tag_fields) { + PyErr_Clear(); + } else if (pyTupleContainsString(*py_tag_fields, name)) { + return true; + } + auto py_indexed_fields = Py_OWN(PyObject_GetAttrString(reinterpret_cast(memo_type), "__DBZERO_INDEXED_FIELDS_ATTR")); + if (!py_indexed_fields) { + PyErr_Clear(); + } else if (pyTupleContainsString(*py_indexed_fields, name)) { + return true; + } + return false; + } + + PyTypeObject *findDeclarationOwner(PyTypeObject *memo_type, const char *name) + { + auto mro = memo_type->tp_mro; + if (mro && PyTuple_Check(mro)) { + auto size = PyTuple_Size(mro); + for (Py_ssize_t index = size - 1; index >= 0; --index) { + auto item = PyTuple_GetItem(mro, index); + if (PyType_Check(item) && PyAnyMemoType_Check(reinterpret_cast(item))) { + auto candidate = reinterpret_cast(item); + if (hasDeclaredName(candidate, name)) { + return candidate; + } + } + } + } + return hasDeclaredName(memo_type, name) ? memo_type : nullptr; + } + + PyObject *tryFieldsOf(PyObject *py_type); + + PyObject *makeFieldRef(PyTypeObject *memo_type, PyTypeObject *owner_type, const char *field_name, bool declared) + { + auto ref = reinterpret_cast(PyFieldRefType.tp_alloc(&PyFieldRefType, 0)); + if (!ref) { + return nullptr; + } + Py_INCREF(memo_type); + Py_INCREF(owner_type); + ref->memo_type = memo_type; + ref->owner_type = owner_type; + ref->field_name = PyUnicode_FromString(field_name); + ref->declared = declared; + if (!ref->field_name) { + Py_DECREF(ref); + return nullptr; + } + return reinterpret_cast(ref); + } + + PyObject *getFieldRef(PyFieldNamespace *ns, const char *field_name, bool explicit_dynamic) + { + PyTypeObject *owner_type = findDeclarationOwner(ns->memo_type, field_name); + bool declared = owner_type != nullptr; + if (!declared && !explicit_dynamic) { + PyErr_Format(PyExc_AttributeError, "%s has no declared memo field %s", ns->memo_type->tp_name, field_name); + return nullptr; + } + if (!declared) { + if (!isPersistentAttrName(field_name)) { + THROWF(db0::InputException) << "Invalid persistent field name: " << field_name; + } + owner_type = ns->memo_type; + } else if (owner_type != ns->memo_type) { + auto owner_ns = Py_OWN(tryFieldsOf(reinterpret_cast(owner_type))); + if (!owner_ns) { + return nullptr; + } + return getFieldRef(reinterpret_cast(*owner_ns), field_name, false); + } + + auto key = Py_OWN(PySafeTuple_Pack( + Py_OWN(PyLong_FromVoidPtr(owner_type)), + Py_OWN(PyUnicode_FromString(field_name)), + Py_OWN(PyBool_FromLong(declared ? 1 : 0)))); + if (!key) { + return nullptr; + } + auto existing = PyDict_GetItemWithError(ns->refs, *key); + if (existing) { + Py_INCREF(existing); + return existing; + } + if (PyErr_Occurred()) { + return nullptr; + } + + auto ref = Py_OWN(makeFieldRef(ns->memo_type, owner_type, field_name, declared)); + if (!ref) { + return nullptr; + } + if (PyDict_SetItem(ns->refs, *key, *ref) < 0) { + return nullptr; + } + return ref.steal(); + } + + PyObject *createNamespace(PyTypeObject *memo_type) + { + auto ns = reinterpret_cast(PyFieldNamespaceType.tp_alloc(&PyFieldNamespaceType, 0)); + if (!ns) { + return nullptr; + } + Py_INCREF(memo_type); + ns->memo_type = memo_type; + ns->refs = PyDict_New(); + if (!ns->refs) { + Py_DECREF(ns); + return nullptr; + } + return reinterpret_cast(ns); + } + + PyObject *tryFieldsOf(PyObject *py_type) + { + if (!PyType_Check(py_type) || !PyAnyMemoType_Check(reinterpret_cast(py_type))) { + PyErr_SetString(PyExc_TypeError, "fields_of requires a dbzero memo type"); + return nullptr; + } + auto memo_type = reinterpret_cast(py_type); + auto existing = PyObject_GetAttrString(py_type, FIELD_NAMESPACE_ATTR); + if (existing) { + return existing; + } + PyErr_Clear(); + + auto ns = Py_OWN(createNamespace(memo_type)); + if (!ns) { + return nullptr; + } + if (PyObject_SetAttrString(py_type, FIELD_NAMESPACE_ATTR, *ns) < 0) { + return nullptr; + } + return ns.steal(); + } + + PyObject *FieldNamespace_getattro(PyFieldNamespace *self, PyObject *attr) + { + auto attr_name = PyUnicode_AsUTF8(attr); + if (!attr_name) { + return nullptr; + } + if (!isDunderName(attr_name)) { + return runSafe(getFieldRef, self, attr_name, false); + } + return PyObject_GenericGetAttr(reinterpret_cast(self), attr); + } + + PyObject *FieldNamespace_subscript(PyFieldNamespace *self, PyObject *key) + { + if (!PyUnicode_Check(key)) { + PyErr_SetString(PyExc_TypeError, "field name must be a string"); + return nullptr; + } + auto field_name = PyUnicode_AsUTF8(key); + if (!field_name) { + return nullptr; + } + return runSafe(getFieldRef, self, field_name, true); + } + + PyObject *FieldNamespace_dir(PyFieldNamespace *self, PyObject *) + { + std::vector names; + auto mro = self->memo_type->tp_mro; + if (mro && PyTuple_Check(mro)) { + auto size = PyTuple_Size(mro); + for (Py_ssize_t index = size - 1; index >= 0; --index) { + auto item = PyTuple_GetItem(mro, index); + if (PyType_Check(item) && PyAnyMemoType_Check(reinterpret_cast(item))) { + appendDecorationNames(reinterpret_cast(item), names); + } + } + } else { + appendDecorationNames(self->memo_type, names); + } + std::sort(names.begin(), names.end()); + + auto result = Py_OWN(PyList_New(names.size())); + Py_ssize_t index = 0; + for (const auto &name: names) { + PySafeList_SetItem(*result, index++, Py_OWN(PyUnicode_FromString(name.c_str()))); + } + return result.steal(); + } + + PyObject *FieldNamespace_reduce(PyObject *, PyObject *) + { + PyErr_SetString(PyExc_TypeError, "FieldNamespace objects cannot be pickled"); + return nullptr; + } + + void FieldNamespace_dealloc(PyFieldNamespace *self) + { + PY_DEALLOC_GUARD(); + Py_XDECREF(self->memo_type); + Py_XDECREF(self->refs); + Py_TYPE(self)->tp_free(reinterpret_cast(self)); + } + + PyObject *FieldNamespace_new(PyTypeObject *, PyObject *, PyObject *) + { + PyErr_SetString(PyExc_TypeError, "FieldNamespace objects are created by dbzero.fields_of"); + return nullptr; + } + + void FieldRef_dealloc(PyFieldRef *self) + { + PY_DEALLOC_GUARD(); + Py_XDECREF(self->memo_type); + Py_XDECREF(self->owner_type); + Py_XDECREF(self->field_name); + Py_TYPE(self)->tp_free(reinterpret_cast(self)); + } + + PyObject *FieldRef_new(PyTypeObject *, PyObject *, PyObject *) + { + PyErr_SetString(PyExc_TypeError, "FieldRef objects are created by dbzero.fields_of"); + return nullptr; + } + + Py_hash_t FieldRef_hash(PyFieldRef *self) + { + Py_hash_t name_hash = PyObject_Hash(self->field_name); + if (name_hash == -1) { + return -1; + } + return name_hash ^ reinterpret_cast(self->owner_type) ^ (self->declared ? 0x9e3779b97f4a7c15ULL : 0); + } + + PyObject *FieldRef_richcompare(PyFieldRef *lhs, PyObject *rhs, int op) + { + if (op != Py_EQ && op != Py_NE) { + Py_RETURN_NOTIMPLEMENTED; + } + bool equal = false; + if (PyFieldRef_Check(rhs)) { + auto other = reinterpret_cast(rhs); + equal = lhs->owner_type == other->owner_type + && lhs->declared == other->declared + && PyObject_RichCompareBool(lhs->field_name, other->field_name, Py_EQ) == 1; + } + if (op == Py_NE) { + equal = !equal; + } + return PyBool_FromLong(equal ? 1 : 0); + } + + PyObject *FieldRef_repr(PyFieldRef *self) + { + auto name = PyUnicode_AsUTF8(self->field_name); + std::stringstream repr; + repr << "owner_type->tp_name << "." << (name ? name : "") << ">"; + return PyUnicode_FromString(repr.str().c_str()); + } + + PyObject *FieldRef_reduce(PyObject *, PyObject *) + { + PyErr_SetString(PyExc_TypeError, "FieldRef objects cannot be pickled"); + return nullptr; + } + + static PyMethodDef FieldNamespace_methods[] = { + {"__dir__", reinterpret_cast(FieldNamespace_dir), METH_NOARGS, "Return declared memo field names"}, + {"__reduce__", reinterpret_cast(FieldNamespace_reduce), METH_NOARGS, ""}, + {NULL} + }; + + static PyMappingMethods FieldNamespace_mapping = { + .mp_length = 0, + .mp_subscript = reinterpret_cast(FieldNamespace_subscript), + .mp_ass_subscript = 0, + }; + + static PyMethodDef FieldRef_methods[] = { + {"__reduce__", reinterpret_cast(FieldRef_reduce), METH_NOARGS, ""}, + {NULL} + }; + } + + PyTypeObject PyFieldNamespaceType = { + PYVAROBJECT_HEAD_INIT_DESIGNATED, + .tp_name = "dbzero.FieldNamespace", + .tp_basicsize = sizeof(PyFieldNamespace), + .tp_itemsize = 0, + .tp_dealloc = reinterpret_cast(FieldNamespace_dealloc), + .tp_as_mapping = &FieldNamespace_mapping, + .tp_getattro = reinterpret_cast(FieldNamespace_getattro), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = "Memo field namespace", + .tp_methods = FieldNamespace_methods, + .tp_alloc = PyType_GenericAlloc, + .tp_new = FieldNamespace_new, + .tp_free = PyObject_Free, + }; + + PyTypeObject PyFieldRefType = { + PYVAROBJECT_HEAD_INIT_DESIGNATED, + .tp_name = "dbzero.FieldRef", + .tp_basicsize = sizeof(PyFieldRef), + .tp_itemsize = 0, + .tp_dealloc = reinterpret_cast(FieldRef_dealloc), + .tp_repr = reinterpret_cast(FieldRef_repr), + .tp_hash = reinterpret_cast(FieldRef_hash), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = "Memo field reference", + .tp_richcompare = reinterpret_cast(FieldRef_richcompare), + .tp_methods = FieldRef_methods, + .tp_alloc = PyType_GenericAlloc, + .tp_new = FieldRef_new, + .tp_free = PyObject_Free, + }; + + bool PyFieldRef_Check(PyObject *py_object) + { + return Py_TYPE(py_object) == &PyFieldRefType; + } + + PyTypeObject *PyFieldRef_getMemoType(PyObject *py_object) + { + return reinterpret_cast(py_object)->memo_type; + } + + const char *PyFieldRef_getFieldName(PyObject *py_object) + { + return PyUnicode_AsUTF8(reinterpret_cast(py_object)->field_name); + } + + PyObject *PyAPI_fieldsOf(PyObject *, PyObject *const *args, Py_ssize_t nargs) + { + PY_API_FUNC + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "fields_of requires exactly one argument"); + return nullptr; + } + return runSafe(tryFieldsOf, args[0]); + } + +} diff --git a/src/dbzero/bindings/python/PyFieldRef.hpp b/src/dbzero/bindings/python/PyFieldRef.hpp new file mode 100644 index 00000000..df602bb0 --- /dev/null +++ b/src/dbzero/bindings/python/PyFieldRef.hpp @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2026 DBZero Software sp. z o.o. + +#pragma once + +#include + +namespace db0::python +{ + + struct PyFieldNamespace { + PyObject_HEAD + PyTypeObject *memo_type = nullptr; + PyObject *refs = nullptr; + }; + + struct PyFieldRef { + PyObject_HEAD + PyTypeObject *memo_type = nullptr; + PyTypeObject *owner_type = nullptr; + PyObject *field_name = nullptr; + bool declared = false; + }; + + extern PyTypeObject PyFieldNamespaceType; + extern PyTypeObject PyFieldRefType; + + bool PyFieldRef_Check(PyObject *); + PyTypeObject *PyFieldRef_getMemoType(PyObject *); + const char *PyFieldRef_getFieldName(PyObject *); + + PyObject *PyAPI_fieldsOf(PyObject *, PyObject *const *args, Py_ssize_t nargs); + +} diff --git a/src/dbzero/bindings/python/PyInternalAPI.cpp b/src/dbzero/bindings/python/PyInternalAPI.cpp index 48169da8..e9262f69 100755 --- a/src/dbzero/bindings/python/PyInternalAPI.cpp +++ b/src/dbzero/bindings/python/PyInternalAPI.cpp @@ -42,6 +42,97 @@ namespace db0::python { + namespace + { + void renameFieldTupleAttribute(PyTypeObject *py_type, const char *attr_name, const char *from_name, + const char *to_name) + { + auto py_fields = Py_OWN(PyObject_GetAttrString(reinterpret_cast(py_type), attr_name)); + if (!py_fields) { + PyErr_Clear(); + return; + } + if (!PyTuple_Check(*py_fields)) { + return; + } + + auto size = PyTuple_Size(*py_fields); + auto py_updated = Py_OWN(PyTuple_New(size)); + if (!py_updated) { + return; + } + + bool changed = false; + for (Py_ssize_t i = 0; i < size; ++i) { + auto item = PyTuple_GetItem(*py_fields, i); + if (!item) { + return; + } + PyObject *new_item = item; + if (PyUnicode_Check(item)) { + auto value = PyUnicode_AsUTF8(item); + if (!value) { + return; + } + if (std::strcmp(value, from_name) == 0) { + new_item = PyUnicode_FromString(to_name); + if (!new_item) { + return; + } + changed = true; + } else { + Py_INCREF(new_item); + } + } else { + Py_INCREF(new_item); + } + if (PyTuple_SetItem(*py_updated, i, new_item) < 0) { + Py_DECREF(new_item); + return; + } + } + if (changed && PyObject_SetAttrString(reinterpret_cast(py_type), attr_name, *py_updated) < 0) { + return; + } + } + } + + + const char *parsePrefixName(PyObject *py_prefix, const char *api_name, const char *arg_name) + { + if (!py_prefix || py_prefix == Py_None) { + return nullptr; + } + if (!PyUnicode_Check(py_prefix)) { + PyErr_Format(PyExc_TypeError, "%s() argument '%s' must be str or None, not %s", + api_name, arg_name, Py_TYPE(py_prefix)->tp_name); + return nullptr; + } + return PyUnicode_AsUTF8(py_prefix); + } + + db0::swine_ptr resolveMemoTypeFixture(PyTypeObject *memo_type, const char *prefix_name) + { + auto &decor = MemoTypeDecoration::get(memo_type); + auto &workspace = PyToolkit::getPyWorkspace().getWorkspace(); + if (prefix_name) { + auto requested_prefix = db0::PrefixName(prefix_name); + if (decor.isScoped() && requested_prefix != decor.getPrefixName()) { + THROWF(db0::InputException) + << "Explicit prefix conflicts with scoped memo type prefix"; + } + auto fixture = workspace.tryFindFixture(requested_prefix); + if (!fixture) { + THROWF(db0::InputException) << "Prefix is not open: " << requested_prefix.c_str(); + } + return fixture; + } + + if (decor.isScoped()) { + return workspace.getFixture(decor.getPrefixName(), db0::AccessType::READ_WRITE); + } + return workspace.getCurrentFixture(); + } namespace { @@ -416,53 +507,9 @@ namespace db0::python type->renameField(from_name, to_name); static constexpr const char *TAG_FIELDS_ATTR = "__DBZERO_TAG_FIELDS_ATTR"; - auto py_tag_fields = Py_OWN(PyObject_GetAttrString(reinterpret_cast(py_type), TAG_FIELDS_ATTR)); - if (!py_tag_fields) { - PyErr_Clear(); - return; - } - if (!PyTuple_Check(*py_tag_fields)) { - return; - } - - auto size = PyTuple_Size(*py_tag_fields); - auto py_updated = Py_OWN(PyTuple_New(size)); - if (!py_updated) { - return; - } - - bool changed = false; - for (Py_ssize_t i = 0; i < size; ++i) { - auto item = PyTuple_GetItem(*py_tag_fields, i); - if (!item) { - return; - } - PyObject *new_item = item; - if (PyUnicode_Check(item)) { - auto value = PyUnicode_AsUTF8(item); - if (!value) { - return; - } - if (std::strcmp(value, from_name) == 0) { - new_item = PyUnicode_FromString(to_name); - if (!new_item) { - return; - } - changed = true; - } else { - Py_INCREF(new_item); - } - } else { - Py_INCREF(new_item); - } - if (PyTuple_SetItem(*py_updated, i, new_item) < 0) { - Py_DECREF(new_item); - return; - } - } - if (changed && PyObject_SetAttrString(reinterpret_cast(py_type), TAG_FIELDS_ATTR, *py_updated) < 0) { - return; - } + static constexpr const char *INDEXED_FIELDS_ATTR = "__DBZERO_INDEXED_FIELDS_ATTR"; + renameFieldTupleAttribute(py_type, TAG_FIELDS_ATTR, from_name, to_name); + renameFieldTupleAttribute(py_type, INDEXED_FIELDS_ATTR, from_name, to_name); } #ifndef NDEBUG @@ -568,7 +615,22 @@ namespace db0::python // DB0_INDEX specialization template <> void dropInstance(PyObject *py_wrapper) { - PyWrapper_drop(reinterpret_cast(py_wrapper)); + auto index_wrapper = reinterpret_cast(py_wrapper); + auto index = index_wrapper->getSharedPtr(); + if (!index || !index->hasInstance()) { + return; + } + if (index->hasRefs()) { + PyErr_SetString(PyExc_RuntimeError, "delete failed: object has references"); + return; + } + + auto fixture = index->getFixture(); + auto address = index->getAddress(); + index_wrapper->modifyExt().destroy(); + fixture->getVObjectCache().erase(address); + index_wrapper->reset(); + fixture->getLangCache().erase(address); } void registerDropInstanceFunctions(std::vector &functions) @@ -577,6 +639,7 @@ namespace db0::python functions.resize(static_cast(TypeId::COUNT)); std::fill(functions.begin(), functions.end(), nullptr); functions[static_cast(TypeId::MEMO_OBJECT)] = dropInstance; + functions[static_cast(TypeId::DB0_INDEX)] = dropInstance; } void dropInstance(db0::bindings::TypeId type_id, PyObject *py_instance) diff --git a/src/dbzero/bindings/python/PyInternalAPI.hpp b/src/dbzero/bindings/python/PyInternalAPI.hpp index 30df0b74..942011ce 100755 --- a/src/dbzero/bindings/python/PyInternalAPI.hpp +++ b/src/dbzero/bindings/python/PyInternalAPI.hpp @@ -97,6 +97,9 @@ namespace db0::python // Check if object exists with optional type validation bool isExistingObject(db0::swine_ptr &fixture, ObjectId object_id, PyTypeObject *py_expected_type = nullptr); + + const char *parsePrefixName(PyObject *py_prefix, const char *api_name, const char *arg_name = "prefix"); + db0::swine_ptr resolveMemoTypeFixture(PyTypeObject *memo_type, const char *prefix_name); void renameMemoClassField(PyTypeObject *py_type, const char *from_name, const char *to_name); diff --git a/src/dbzero/bindings/python/PyToolkit.cpp b/src/dbzero/bindings/python/PyToolkit.cpp index 419718ab..6d6cfd95 100755 --- a/src/dbzero/bindings/python/PyToolkit.cpp +++ b/src/dbzero/bindings/python/PyToolkit.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -236,6 +237,17 @@ namespace db0::python return getTypeManager().extractAnyObject(pyObject).getUniqueAddress(); } + std::optional PyToolkit::tryGetMemoUniqueAddress(ObjectPtr pyObject) + { + if (PyMemo_Check(pyObject)) { + return db0::object_model::getMemoUniqueAddress(reinterpret_cast(pyObject)); + } + if (PyMemo_Check(pyObject)) { + return db0::object_model::getMemoUniqueAddress(reinterpret_cast(pyObject)); + } + return {}; + } + bool PyToolkit::isMemoDead(ObjectPtr pyObject) { if (PyEmbeddedMemo_Check(pyObject)) { @@ -881,8 +893,10 @@ namespace db0::python } auto py_index = Py_OWN(IndexDefaultObject_new()); - // retrieve actual dbzero instance - py_index->unload(fixture, address, access_mode); + auto index = fixture->getVObjectCache().findOrPull( + address, true, fixture, address, access_mode + ); + py_index->makeNew(index); // add list object to cache // NOTE: in case of Index (which requires a flush on update) we need to cache instance @@ -896,7 +910,7 @@ namespace db0::python // Keep the callback's own ref balance explicit so an unmatched clean // does not drop the LangCache-owned Index wrapper. auto dirty_ref_count = std::make_shared(0); - py_index->ext().setDirtyCallback([py_index_ptr, dirty_ref_count](bool incRef) { + py_index->modifyExt().setDirtyCallback([py_index_ptr, dirty_ref_count](bool incRef) { if (incRef) { Py_INCREF(py_index_ptr); ++(*dirty_ref_count); @@ -1255,30 +1269,48 @@ namespace db0::python return MemoTypeDecoration::get(memo_type).getInitVars(); } - std::vector PyToolkit::getTagFields(TypeObjectPtr memo_type) + namespace { - assert(isAnyMemoType(memo_type)); - auto py_tag_fields = Py_OWN(PyObject_GetAttrString(reinterpret_cast(memo_type), - "__DBZERO_TAG_FIELDS_ATTR")); - if (!py_tag_fields) { - PyErr_Clear(); - return MemoTypeDecoration::get(memo_type).getTagFields(); - } - if (!PyTuple_Check(*py_tag_fields)) { - return MemoTypeDecoration::get(memo_type).getTagFields(); - } + std::vector getStringTupleAttributeOr(PyToolkit::TypeObjectPtr memo_type, + const char *attribute_name, const std::vector &fallback) + { + auto py_fields = Py_OWN(PyObject_GetAttrString(reinterpret_cast(memo_type), attribute_name)); + if (!py_fields) { + PyErr_Clear(); + return fallback; + } + if (!PyTuple_Check(*py_fields)) { + return fallback; + } - std::vector result; - auto size = PyTuple_Size(*py_tag_fields); - result.reserve(size); - for (Py_ssize_t index = 0; index < size; ++index) { - auto item = PyTuple_GetItem(*py_tag_fields, index); - if (!PyUnicode_Check(item)) { - return MemoTypeDecoration::get(memo_type).getTagFields(); + std::vector result; + auto size = PyTuple_Size(*py_fields); + result.reserve(size); + for (Py_ssize_t index = 0; index < size; ++index) { + auto item = PyTuple_GetItem(*py_fields, index); + if (!PyUnicode_Check(item)) { + return fallback; + } + result.emplace_back(PyUnicode_AsUTF8(item)); } - result.emplace_back(PyUnicode_AsUTF8(item)); + return result; } - return result; + } + + std::vector PyToolkit::getTagFields(TypeObjectPtr memo_type) + { + assert(isAnyMemoType(memo_type)); + return getStringTupleAttributeOr( + memo_type, "__DBZERO_TAG_FIELDS_ATTR", MemoTypeDecoration::get(memo_type).getTagFields() + ); + } + + std::vector PyToolkit::getIndexedFields(TypeObjectPtr memo_type) + { + assert(isAnyMemoType(memo_type)); + return getStringTupleAttributeOr( + memo_type, "__DBZERO_INDEXED_FIELDS_ATTR", MemoTypeDecoration::get(memo_type).getIndexedFields() + ); } bool PyToolkit::isAnyMemoType(TypeObjectPtr py_type) { diff --git a/src/dbzero/bindings/python/PyToolkit.hpp b/src/dbzero/bindings/python/PyToolkit.hpp index bf9b7b1a..fbb70b01 100755 --- a/src/dbzero/bindings/python/PyToolkit.hpp +++ b/src/dbzero/bindings/python/PyToolkit.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include "PyTypeManager.hpp" #include "PyWorkspace.hpp" #include "PyTypes.hpp" @@ -36,6 +37,7 @@ namespace db0::object_model class o_py_tuple; class Object; class ObjectIterable; + class Index; class Class; class ClassFactory; struct EnumValue; @@ -76,8 +78,12 @@ namespace db0::python return m_py_workspace; } - template inline static PyWrapper *getWrapperTypeOf(ObjectPtr ptr) { - return static_cast *>(ptr); + template inline static auto *getWrapperTypeOf(ObjectPtr ptr) { + if constexpr (std::is_same_v) { + return static_cast *>(ptr); + } else { + return static_cast *>(ptr); + } } /** @@ -218,6 +224,7 @@ namespace db0::python static const char *getMemoTypeID(TypeObjectPtr memo_type); static const std::vector &getInitVars(TypeObjectPtr memo_type); static std::vector getTagFields(TypeObjectPtr memo_type); + static std::vector getIndexedFields(TypeObjectPtr memo_type); static bool isSingleton(TypeObjectPtr); // check if a memo type is marked with no_default_tags flag @@ -230,6 +237,7 @@ namespace db0::python static FlagSet getMemoFlags(TypeObjectPtr); static bool hasMemoInstance(ObjectPtr); static UniqueAddress getMemoUniqueAddress(ObjectPtr); + static std::optional tryGetMemoUniqueAddress(ObjectPtr); static bool isMemoDead(ObjectPtr); static bool isMemoDropped(ObjectPtr); static bool hasMemoAnyRefs(ObjectPtr); diff --git a/src/dbzero/bindings/python/collections/PyIndex.cpp b/src/dbzero/bindings/python/collections/PyIndex.cpp index 8e9c0a01..d37e3efa 100755 --- a/src/dbzero/bindings/python/collections/PyIndex.cpp +++ b/src/dbzero/bindings/python/collections/PyIndex.cpp @@ -2,13 +2,49 @@ // Copyright (c) 2025 DBZero Software sp. z o.o. #include "PyIndex.hpp" +#include #include #include +#include #include +#include namespace db0::python { + namespace + { + void requireMutableIndex(const IndexObject *index_obj) + { + if (index_obj->ext().isManaged()) { + THROWF(db0::InputException) << "managed indexes are read-only"; + } + } + + std::shared_ptr getManagedIndexForRead(const IndexObject *index_obj) + { + auto &index = index_obj->ext(); + if (!index.isManaged() || !index.hasRefs()) { + return nullptr; + } + auto fixture = index.getFixture(); + return fixture->getVObjectCache().findOrPull( + index.getAddress(), true, fixture, index.getAddress() + ); + } + } + + bool isValidIndexedFieldName(PyTypeObject *memo_type, const char *field_name) + { + for (auto type = memo_type; type && PyAnyMemoType_Check(type); type = PyToolkit::getBaseMemoType(type)) { + auto fields = PyToolkit::getIndexedFields(type); + if (std::find(fields.begin(), fields.end(), field_name) != fields.end()) { + return true; + } + } + return false; + } + static PyMethodDef IndexObject_methods[] = { {"add", (PyCFunction)PyAPI_IndexObject_add, METH_FASTCALL, "Add item to index."}, @@ -47,6 +83,17 @@ namespace db0::python IndexObject *IndexDefaultObject_new() { return IndexObject_new(&IndexObjectType, NULL, NULL); } + + IndexObject *IndexDefaultObject_new(db0::swine_ptr fixture, bool passive, bool managed) + { + auto py_index = IndexDefaultObject_new(); + auto index = fixture->getVObjectCache().pull(true, fixture, passive); + if (managed) { + index->setManaged(); + } + py_index->makeNew(index); + return py_index; + } void PyAPI_IndexObject_del(IndexObject* index_obj) { @@ -60,21 +107,28 @@ namespace db0::python Py_ssize_t tryIndexObject_len(IndexObject *index_obj) { index_obj->ext().getFixture()->refreshIfUpdated(); - return index_obj->ext().size(); + auto managed_index = getManagedIndexForRead(index_obj); + return managed_index ? managed_index->size() : index_obj->ext().size(); } Py_ssize_t PyAPI_IndexObject_len(IndexObject *index_obj) { PY_API_FUNC - return runSafe(tryIndexObject_len, index_obj); + return runSafe<-1>(tryIndexObject_len, index_obj); } - IndexObject *tryMakeIndex(PyObject *self, PyObject *const *args, Py_ssize_t nargs) + IndexObject *tryMakeIndex(PyObject *self, PyObject *args, PyObject *kwargs) { + static const char *kwlist[] = {"passive", NULL}; + int passive = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|$p", const_cast(kwlist), &passive)) { + return nullptr; + } + // make actual dbzero instance, use default fixture - auto py_index = Py_OWN(IndexDefaultObject_new()); db0::FixtureLock lock(PyToolkit::getPyWorkspace().getWorkspace().getCurrentFixture()); - auto &index = py_index->makeNew(*lock); + auto py_index = Py_OWN(IndexDefaultObject_new(*lock, passive != 0)); + auto &index = py_index->modifyExt(); // NOTE: this callback is important for proper lifecycle management // we must prevent dirty Index instance from deletion @@ -102,18 +156,15 @@ namespace db0::python return py_index.steal(); } - IndexObject *PyAPI_makeIndex(PyObject *self, PyObject *const *args, Py_ssize_t nargs) + PyObject *PyAPI_makeIndex(PyObject *self, PyObject *args, PyObject *kwargs) { - if (nargs != 0) { - PyErr_SetString(PyExc_TypeError, "Index object does not accept arguments"); - return NULL; - } PY_API_FUNC - return runSafe(tryMakeIndex, self, args, nargs); + return runSafe(tryMakeIndex, self, args, kwargs); } PyObject *tryIndexObject_add(IndexObject *index_obj, PyObject *const *args, Py_ssize_t nargs) { + requireMutableIndex(index_obj); index_obj->modifyExt().add(args[0], args[1]); // NOTE: we don't need to lock the fixture here, because add() is a buffered operation index_obj->ext().getFixture()->onUpdated(); @@ -133,6 +184,7 @@ namespace db0::python PyObject *tryIndexObject_remove(IndexObject *index_obj, PyObject *const *args, Py_ssize_t nargs) { + requireMutableIndex(index_obj); index_obj->modifyExt().remove(args[0], args[1]); // NOTE: we don't need to lock the fixture here, because remove() is a buffered operation index_obj->ext().getFixture()->onUpdated(); @@ -171,9 +223,10 @@ namespace db0::python return NULL; } - auto &index = py_index->ext(); + auto managed_index = getManagedIndexForRead(py_index); + auto *index = managed_index ? managed_index.get() : &py_index->ext(); auto &iter = reinterpret_cast(py_iter)->modifyExt(); - auto iter_sorted = index.sort(iter, asc, null_first); + auto iter_sorted = index->sort(iter, asc, null_first); auto iter_obj = PyObjectIterableDefault_new(); iter_obj->makeNew(iter, std::move(iter_sorted)); return iter_obj.steal(); @@ -204,14 +257,19 @@ namespace db0::python return NULL; } - auto &index = py_index->ext(); + auto managed_index = getManagedIndexForRead(py_index); + auto *index = managed_index ? managed_index.get() : &py_index->ext(); // construct range iterator - auto iter_factory = index.range(low, high, null_first); - auto fixture = index.getFixture(); + auto iter_factory = index->range(low, high, null_first); + auto fixture = index->getFixture(); auto py_iter_obj = PyObjectIterableDefault_new(); + ObjectIterable::QueryPlanning query_planning { + index->isPassive(), + !index->isPassive() + }; py_iter_obj->makeNew( fixture, std::move(iter_factory), nullptr, nullptr, std::vector >{}, - std::vector{} + std::vector{}, query_planning ); return py_iter_obj.steal(); } @@ -228,6 +286,7 @@ namespace db0::python PyObject *tryIndexObject_flush(IndexObject *self) { + requireMutableIndex(self); FixtureLock lock(self->ext().getFixture()); self->modifyExt().flush(lock); Py_RETURN_NONE; @@ -241,6 +300,7 @@ namespace db0::python PyObject *tryIndexObject_clear(IndexObject *self) { + requireMutableIndex(self); FixtureLock lock(self->ext().getFixture()); self->modifyExt().clear(lock); Py_RETURN_NONE; diff --git a/src/dbzero/bindings/python/collections/PyIndex.hpp b/src/dbzero/bindings/python/collections/PyIndex.hpp index 9b224a96..495e9156 100755 --- a/src/dbzero/bindings/python/collections/PyIndex.hpp +++ b/src/dbzero/bindings/python/collections/PyIndex.hpp @@ -5,15 +5,17 @@ #include #include +#include namespace db0::python { - using IndexObject = PyWrapper; + using IndexObject = PySharedWrapper; IndexObject *IndexObject_new(PyTypeObject *type, PyObject *, PyObject *); IndexObject* IndexDefaultObject_new(); + IndexObject* IndexDefaultObject_new(db0::swine_ptr fixture, bool passive, bool managed = false); void PyAPI_IndexObject_del(IndexObject* self); Py_ssize_t PyAPI_IndexObject_len(IndexObject *); @@ -27,7 +29,8 @@ namespace db0::python extern PyTypeObject IndexObjectType; - IndexObject *PyAPI_makeIndex(PyObject *self, PyObject *const *args, Py_ssize_t nargs); + PyObject *PyAPI_makeIndex(PyObject *self, PyObject *args, PyObject *kwargs); bool IndexObject_Check(PyObject *); + bool isValidIndexedFieldName(PyTypeObject *memo_type, const char *field_name); -} \ No newline at end of file +} diff --git a/src/dbzero/bindings/python/dbzero.cpp b/src/dbzero/bindings/python/dbzero.cpp index 894291f4..e7791c81 100755 --- a/src/dbzero/bindings/python/dbzero.cpp +++ b/src/dbzero/bindings/python/dbzero.cpp @@ -17,6 +17,7 @@ #include "PyLocked.hpp" #include "PyWeakProxy.hpp" #include "MigrateError.hpp" +#include "PyFieldRef.hpp" #include #include #include @@ -63,7 +64,9 @@ static PyMethodDef dbzero_methods[] = {"uuid", (PyCFunction)&py::PyAPI_getUUID, METH_FASTCALL, "Get unique object ID"}, {"clear_cache", &py::PyAPI_clearCache, METH_NOARGS, "Clear dbzero cache"}, {"list", (PyCFunction)&py::PyAPI_makeList, METH_FASTCALL, "Create a new dbzero list instance"}, - {"index", (PyCFunction)&py::PyAPI_makeIndex, METH_FASTCALL, "Create a new dbzero index instance"}, + {"index", (PyCFunction)&py::PyAPI_makeIndex, METH_VARARGS | METH_KEYWORDS, "Create a new dbzero index instance"}, + {"index_of", (PyCFunction)&py::PyAPI_indexOf, METH_VARARGS | METH_KEYWORDS, "Get the managed index for an indexed memo field"}, + {"fields_of", (PyCFunction)&py::PyAPI_fieldsOf, METH_FASTCALL, "Get a memo field-reference namespace"}, {"tuple", (PyCFunction)&py::PyAPI_makeTuple, METH_FASTCALL, "Create a new dbzero tuple instance"}, {"set", (PyCFunction)&py::PyAPI_makeSet, METH_FASTCALL, "Create a new dbzero set instance"}, {"weak_set", (PyCFunction)&py::PyAPI_makeWeakSet, METH_FASTCALL, "Create a new dbzero weak set instance"}, @@ -108,6 +111,7 @@ static PyMethodDef dbzero_methods[] = {"hash", (PyCFunction)&py::PyAPI_hash, METH_FASTCALL, "Returns hash of python or db0 object"}, {"as_tag", (PyCFunction)&py::PyAPI_as_tag, METH_FASTCALL, "Returns tag of a @db0.memo object"}, {"_get_tag_fields", (PyCFunction)&py::PyAPI_getTagFields, METH_FASTCALL, "Get materialized tag field names for a memo class"}, + {"_get_indexed_fields", (PyCFunction)&py::PyAPI_getIndexedFields, METH_FASTCALL, "Get materialized indexed field names for a memo class"}, {"materialized", (PyCFunction)&py::PyAPI_materialized, METH_FASTCALL, "Returns a materialized version of a @db0.memo object"}, {"is_memo", (PyCFunction)&py::PyAPI_PyMemo_Check, METH_FASTCALL, "Checks if passed object is memo type"}, {"is_enum", (PyCFunction)&py::PyAPI_isEnum, METH_FASTCALL, "Checks if passed object is a db0 enum value"}, @@ -247,6 +251,8 @@ PyMODINIT_FUNC PyInit_dbzero(void) &py::PyEnumValueReprType, &py::PyClassFieldsType, &py::PyFieldDefType, + &py::PyFieldNamespaceType, + &py::PyFieldRefType, &py::ClassObjectType, &py::TagSetType, &py::PyAtomicType, diff --git a/src/dbzero/bindings/python/iter/PyObjectIterable.cpp b/src/dbzero/bindings/python/iter/PyObjectIterable.cpp index 89e3cdab..977f7273 100755 --- a/src/dbzero/bindings/python/iter/PyObjectIterable.cpp +++ b/src/dbzero/bindings/python/iter/PyObjectIterable.cpp @@ -72,6 +72,7 @@ namespace db0::python PyObject *tryPyAPI_PyObjectIterable_iter(PyObjectIterable *py_iterable) { + py_iterable->ext().requirePassiveAnchor(); // getFixture to prevent segfault in case the associated context (e.g. snapshot) has been destroyed auto fixture = py_iterable->ext().getFixture(); auto py_iter = PyObjectIteratorDefault_new(); @@ -90,6 +91,7 @@ namespace db0::python Py_ssize_t tryPyObjectIterable_len(PyObjectIterable *py_iterable) { + py_iterable->ext().requirePassiveAnchor(); // getFixture to prevent segfault in case the associated context (e.g. snapshot) has been destroyed py_iterable->ext().getFixture(); return py_iterable->ext().getSize(); @@ -160,6 +162,7 @@ namespace db0::python using SliceDef = db0::object_model::SliceDef; using ObjectSharedPtr = PyToolkit::ObjectSharedPtr; + py_iterable->ext().requirePassiveAnchor(); if (PyTuple_Check(py_key)) { // itemgetter's key (item indexes) auto indices = unpackTuple(py_key); @@ -208,6 +211,7 @@ namespace db0::python int PyAPI_PyObjectIterable_bool(PyObjectIterable *py_iterable) { PY_API_FUNC + py_iterable->ext().requirePassiveAnchor(); // check if the iterable is empty if (py_iterable->ext().empty()) { return 0; // False diff --git a/src/dbzero/bindings/python/types/PyObjectId.hpp b/src/dbzero/bindings/python/types/PyObjectId.hpp index 20b9e145..02e78295 100755 --- a/src/dbzero/bindings/python/types/PyObjectId.hpp +++ b/src/dbzero/bindings/python/types/PyObjectId.hpp @@ -26,7 +26,7 @@ namespace db0::python { using ListObject = PyWrapper; - using IndexObject = PyWrapper; + using IndexObject = PySharedWrapper; using ObjectId = db0::object_model::ObjectId; struct PyObjectId diff --git a/src/dbzero/core/collections/range_tree/IndexBase.cpp b/src/dbzero/core/collections/range_tree/IndexBase.cpp index f0bac900..f20bbb54 100755 --- a/src/dbzero/core/collections/range_tree/IndexBase.cpp +++ b/src/dbzero/core/collections/range_tree/IndexBase.cpp @@ -3,6 +3,8 @@ #include "IndexBase.hpp" +DEFINE_ENUM_VALUES(db0::IndexOptions, "Passive", "Managed") + namespace db0 { @@ -18,7 +20,43 @@ namespace db0 o_index::o_index(const o_index &other) : m_type(other.m_type) , m_data_type(other.m_data_type) + , m_flags(other.m_flags) + { + } + + bool o_index::isPassive() const + { + return getObjVer() >= 1 && m_flags[IndexOptions::Passive]; + } + + bool o_index::isManaged() const + { + return getObjVer() >= 1 && m_flags[IndexOptions::Managed]; + } + + void o_index::setManaged() + { + if (getObjVer() < 1) { + return; + } + m_flags.set(IndexOptions::Passive); + m_flags.set(IndexOptions::Managed); + } + + bool isSupportedIndexKeyType(TypeId type_id) { + switch (type_id) { + case TypeId::INTEGER: + case TypeId::DATETIME: + case TypeId::DATETIME_TZ: + case TypeId::DATE: + case TypeId::TIME: + case TypeId::TIME_TZ: + case TypeId::DECIMAL: + return true; + default: + return false; + } } IndexDataType getIndexDataType(TypeId type_id) @@ -39,4 +77,4 @@ namespace db0 } } -} \ No newline at end of file +} diff --git a/src/dbzero/core/collections/range_tree/IndexBase.hpp b/src/dbzero/core/collections/range_tree/IndexBase.hpp index d2a8c050..05b36a60 100755 --- a/src/dbzero/core/collections/range_tree/IndexBase.hpp +++ b/src/dbzero/core/collections/range_tree/IndexBase.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace db0 @@ -31,9 +32,25 @@ namespace db0 Int64 = 2, UInt64 = 3 }; + + enum class IndexOptions: std::uint16_t + { + Passive = 0x0001, + Managed = 0x0002 + }; + + using IndexFlags = db0::FlagSet; + +} + +DECLARE_ENUM_VALUES(db0::IndexOptions, 2) + +namespace db0 + +{ DB0_PACKED_BEGIN - struct DB0_PACKED_ATTR o_index: public o_fixed_versioned + struct DB0_PACKED_ATTR o_index: public o_fixed_versioned { // common object header o_unique_header m_header; @@ -41,6 +58,7 @@ DB0_PACKED_BEGIN IndexDataType m_data_type = IndexDataType::Auto; // address of the actual index instance Address m_index_addr = {}; + IndexFlags m_flags; o_index(IndexType, IndexDataType); // header not copied @@ -49,11 +67,16 @@ DB0_PACKED_BEGIN bool hasRefs() const { return m_header.hasRefs(); } + + bool isPassive() const; + bool isManaged() const; + void setManaged(); }; DB0_PACKED_END using IndexBase = db0::v_object; + bool isSupportedIndexKeyType(db0::bindings::TypeId); IndexDataType getIndexDataType(db0::bindings::TypeId); template std::shared_ptr tryGetRangeTree(IndexBase &index) @@ -66,4 +89,4 @@ DB0_PACKED_END return std::make_shared(index.myPtr(index->m_index_addr)); } -} \ No newline at end of file +} diff --git a/src/dbzero/object_model/Utils.hpp b/src/dbzero/object_model/Utils.hpp new file mode 100644 index 00000000..c3ef6a58 --- /dev/null +++ b/src/dbzero/object_model/Utils.hpp @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2026 DBZero Software sp. z o.o. + +#pragma once + +#include + +namespace db0::object_model + +{ + + template + UniqueAddress getMemoUniqueAddress(MemoT *memo_obj) + { + return memo_obj->ext().getUniqueAddress(); + } + +} diff --git a/src/dbzero/object_model/class/Class.cpp b/src/dbzero/object_model/class/Class.cpp index 6d7cbfc9..164d6a36 100755 --- a/src/dbzero/object_model/class/Class.cpp +++ b/src/dbzero/object_model/class/Class.cpp @@ -9,9 +9,11 @@ #include #include #include +#include #include "Schema.hpp" DEFINE_ENUM_VALUES(db0::ClassOptions, "SINGLETON", "NO_DEFAULT_TAGS", "IMMUTABLE", "RESERVED_0008", "RESERVED_0010", "ACCESS_CONTROL") +DEFINE_ENUM_VALUES(db0::object_model::FieldOptions, "TAG_FIELD", "INDEXED_FIELD") namespace db0::object_model @@ -19,6 +21,25 @@ namespace db0::object_model using namespace db0; using namespace db0::pools; + + template + void setFieldOption(std::unordered_map &options_by_field, const KeyT &key, + FieldOptions option, bool enabled) + { + auto it = options_by_field.find(key); + if (enabled) { + if (it == options_by_field.end()) { + options_by_field.emplace(key, FieldFlags({ option })); + } else { + it->second.set(option); + } + } else if (it != options_by_field.end()) { + it->second.clear(option); + if (it->second.value() == 0) { + options_by_field.erase(it); + } + } + } GC0_Define(Class) @@ -73,17 +94,19 @@ namespace db0::object_model } } - Class::Member::Member(FieldID field_id, unsigned int fidelity, const char *name) + Class::Member::Member(FieldID field_id, unsigned int fidelity, const char *name, FieldFlags field_options) : m_field_id(field_id) , m_fidelity(fidelity) , m_name(name) + , m_field_options(field_options) { } - Class::Member::Member(FieldID field_id, unsigned int fidelity, const std::string &name) + Class::Member::Member(FieldID field_id, unsigned int fidelity, const std::string &name, FieldFlags field_options) : m_field_id(field_id) , m_fidelity(fidelity) , m_name(name) + , m_field_options(field_options) { } @@ -115,6 +138,9 @@ namespace db0::object_model m_tag_fields.setKeyChangeCallback([this](const FieldID &field_id, bool added) { onTagFieldKeyChange(field_id, added); }); + m_indexed_fields.setKeyChangeCallback([this](const IndexedField &indexed_field, bool added) { + onIndexedFieldKeyChange(indexed_field, added); + }); openTagFields(); if (hasOwnAccessControl()) { setAccessControl(); @@ -134,6 +160,9 @@ namespace db0::object_model m_tag_fields.setKeyChangeCallback([this](const FieldID &field_id, bool added) { onTagFieldKeyChange(field_id, added); }); + m_indexed_fields.setKeyChangeCallback([this](const IndexedField &indexed_field, bool added) { + onIndexedFieldKeyChange(indexed_field, added); + }); openTagFields(); m_schema.postInit(getTotalFunc()); // initialize base class if such exists @@ -178,7 +207,17 @@ namespace db0::object_model MemberID Class::addField(const char *name, unsigned int fidelity, bool declared_tag_field) { - return addFieldInternal(name, fidelity, true, declared_tag_field); + auto member_id = addFieldInternal(name, fidelity, true, declared_tag_field); + if (isDeclaredIndexedField(name)) { + if (m_base_class_ptr + && (m_base_class_ptr->isDeclaredIndexedField(name) || m_base_class_ptr->tryGetFieldIndex(name))) { + THROWF(db0::InputException) + << "Indexed field declaration overlaps an ancestor indexed field in class " + << getName(); + } + addIndexedField(member_id.primary().first); + } + return member_id; } MemberID Class::addFieldInternal(const char *name, unsigned int fidelity, bool, bool declared_tag_field) @@ -368,6 +407,9 @@ namespace db0::object_model } else { // extend existing member ID // possibly another fidelity was added + if (it->second.first.tryGet(member.m_fidelity) == member.m_field_id) { + return; + } it->second.first.assign(member.m_field_id, member.m_fidelity); onMemberIDUpdated(it->second.first); } @@ -376,19 +418,45 @@ namespace db0::object_model void Class::openTagFields() const { - if ((*this)->m_reserved_0008_ptr && m_tag_fields.isNull()) { - m_tag_fields.init(getFixture()->myPtr((*this)->m_reserved_0008_ptr.getAddress())); + if ((*this)->m_tag_fields_ptr && m_tag_fields.isNull()) { + m_tag_fields.init(getFixture()->myPtr((*this)->m_tag_fields_ptr.getAddress())); + } + } + + void Class::openIndexedFields() const + { + if ((*this)->getObjVer() >= 1 + && (*this)->m_flags[ClassOptions::RESERVED_0010] + && (*this)->m_indexed_fields_ptr + && m_indexed_fields.isNull()) { + m_indexed_fields.init(getFixture()->myPtr((*this)->m_indexed_fields_ptr.getAddress())); + for (const auto &indexed_field: m_indexed_fields.cached()) { + onIndexedFieldKeyChange(indexed_field, true); + } } } void Class::onTagFieldKeyChange(const FieldID &field_id, bool added) const { + setFieldOption(m_field_options, field_id.getLongIndex(), FieldOptions::TAG_FIELD, added); + reloadMemberOptions(field_id); + } + + void Class::onIndexedFieldKeyChange(const IndexedField &indexed_field, bool added) const + { + auto field_id = indexed_field.getFieldId(); if (added) { - m_tag_field_id_set.insert(field_id.getLongIndex()); + auto fixture = getFixture(); + auto address = indexed_field.m_index_ptr.getAddress(); + m_index_cache[indexed_field.m_field_id] = fixture->getVObjectCache().findOrPull( + address, true, fixture, address + ); + setFieldOption(m_field_options, field_id.getLongIndex(), FieldOptions::INDEXED_FIELD, true); } else { - m_tag_field_id_set.erase(field_id.getLongIndex()); + m_index_cache.erase(indexed_field.m_field_id); + setFieldOption(m_field_options, field_id.getLongIndex(), FieldOptions::INDEXED_FIELD, false); } - m_has_any_tag_fields = !m_tag_field_id_set.empty(); + reloadMemberOptions(field_id); } VTagFields &Class::ensureTagFields() @@ -399,11 +467,27 @@ namespace db0::object_model openTagFields(); if (m_tag_fields.isNull()) { m_tag_fields.init(*getFixture()); - modify().m_reserved_0008_ptr = m_tag_fields; + modify().m_tag_fields_ptr = m_tag_fields; } return m_tag_fields; } + VIndexedFields &Class::ensureIndexedFields() + { + if (!m_indexed_fields.isNull()) { + return m_indexed_fields; + } + if (hasDeclaredIndexedFields()) { + openIndexedFields(); + } + if (m_indexed_fields.isNull()) { + m_indexed_fields.init(*getFixture()); + modify().m_indexed_fields_ptr = m_indexed_fields; + modify().m_flags.set(ClassOptions::RESERVED_0010); + } + return m_indexed_fields; + } + void Class::addTagField(FieldID field_id) { auto &tag_fields = ensureTagFields(); @@ -439,10 +523,154 @@ namespace db0::object_model bool Class::isTagField(FieldID field_id) const { - if (!m_has_any_tag_fields) { - return false; + return getOwnFieldOptions(field_id)[FieldOptions::TAG_FIELD]; + } + + void Class::addIndexedField(FieldID field_id) + { + auto &indexed_fields = ensureIndexedFields(); + for (const auto &indexed_field: indexed_fields.cached()) { + if (indexed_field.getFieldId() == field_id) { + onIndexedFieldKeyChange(indexed_field, true); + return; + } + } + + auto fixture = getFixture(); + Index index(fixture, true); + index.setManaged(); + index.incRef(false); + IndexedField indexed_field(field_id, db0_ptr(index)); + indexed_fields.push_back(indexed_field); + onIndexedFieldKeyChange(indexed_field, true); + } + + void Class::addIndexedField(FieldID field_id, const db0_ptr &index_ptr) + { + auto &indexed_fields = ensureIndexedFields(); + for (const auto &indexed_field: indexed_fields.cached()) { + if (indexed_field.getFieldId() == field_id) { + onIndexedFieldKeyChange(indexed_field, true); + return; + } + } + IndexedField indexed_field(field_id, index_ptr); + indexed_fields.push_back(indexed_field); + onIndexedFieldKeyChange(indexed_field, true); + } + + void Class::removeIndexedField(FieldID field_id) + { + auto &indexed_fields = ensureIndexedFields(); + const auto &cached_indexed_fields = indexed_fields.cached(); + for (std::size_t index = 0; index < cached_indexed_fields.size(); ++index) { + if (cached_indexed_fields[index].getFieldId() == field_id) { + auto indexed_field = cached_indexed_fields[index]; + auto fixture = getFixture(); + auto index_address = indexed_field.m_index_ptr.getAddress(); + auto managed_index_ptr = fixture->getVObjectCache().findOrPull( + index_address, true, fixture, index_address + ); + indexed_fields.erase(index); + fixture->getLangCache().erase(index_address); + if (managed_index_ptr->decRef(false)) { + managed_index_ptr->destroy(); + fixture->getVObjectCache().erase(index_address); + } + onIndexedFieldKeyChange(indexed_field, false); + return; + } + } + } + + const std::vector &Class::getIndexedFieldRecords() const + { + static const std::vector empty_indexed_fields; + openIndexedFields(); + if (m_indexed_fields.isNull()) { + return empty_indexed_fields; + } + return m_indexed_fields.cached(); + } + + std::vector Class::getIndexedFieldIds() const + { + std::vector result; + const auto &records = getIndexedFieldRecords(); + result.reserve(records.size()); + for (const auto &record: records) { + result.push_back(record.getFieldId()); + } + return result; + } + + std::shared_ptr Class::tryGetOwnIndexedFieldIndex(FieldID field_id) const + { + openIndexedFields(); + auto long_index = field_id.getLongIndex(); + auto it = m_index_cache.find(long_index); + if (it != m_index_cache.end()) { + if (auto index = it->second.lock()) { + return index; + } + m_index_cache.erase(it); + } + + if (!m_indexed_fields.isNull()) { + for (const auto &indexed_field: m_indexed_fields.cached()) { + if (indexed_field.m_field_id == long_index) { + auto fixture = getFixture(); + auto address = indexed_field.m_index_ptr.getAddress(); + auto index = fixture->getVObjectCache().findOrPull(address, true, fixture, address); + m_index_cache[long_index] = index; + return index; + } + } + } + return nullptr; + } + + std::shared_ptr Class::tryGetFieldIndex(FieldID field_id) const + { + auto index = tryGetOwnIndexedFieldIndex(field_id); + if (index) { + return index; + } + if (m_base_class_ptr) { + return m_base_class_ptr->tryGetFieldIndex(field_id); + } + return nullptr; + } + + std::shared_ptr Class::tryGetFieldIndex(const char *field_name) const + { + auto member_loc = findField(field_name); + if (!!member_loc.first) { + auto index = tryGetOwnIndexedFieldIndex(member_loc.first.primary().first); + if (index) { + return index; + } + } + if (m_base_class_ptr) { + return m_base_class_ptr->tryGetFieldIndex(field_name); + } + return nullptr; + } + + std::shared_ptr Class::getExistingFieldIndex(FieldID field_id) const + { + auto index = tryGetFieldIndex(field_id); + if (!index) { + auto member = tryGetMember(field_id); + if (member && isDeclaredIndexedField(member->m_name.c_str())) { + const_cast(this)->addIndexedField(field_id); + index = tryGetFieldIndex(field_id); + } + } + if (!index) { + THROWF(db0::InputException) << "Field is not an indexed field"; } - return m_tag_field_id_set.find(field_id.getLongIndex()) != m_tag_field_id_set.end(); + return index; } std::string Class::getTypeName() const { @@ -560,6 +788,7 @@ namespace db0::object_model m_fidelities.detach(); m_schema.detach(); m_tag_fields.detach(); + m_indexed_fields.detach(); super_t::detach(); } @@ -581,6 +810,7 @@ namespace db0::object_model m_fidelities.commit(); m_schema.commit(); m_tag_fields.commit(); + m_indexed_fields.commit(); super_t::commit(); } @@ -695,16 +925,113 @@ namespace db0::object_model void Class::setDeclaredTagFields(const std::vector &tag_fields) { - m_declared_tag_field_set.clear(); - m_declared_tag_field_set.insert(tag_fields.begin(), tag_fields.end()); + for (auto it = m_declared_field_options.begin(); it != m_declared_field_options.end();) { + it->second.clear(FieldOptions::TAG_FIELD); + if (it->second.value() == 0) { + it = m_declared_field_options.erase(it); + } else { + ++it; + } + } + for (const auto &field_name: tag_fields) { + setFieldOption(m_declared_field_options, field_name, FieldOptions::TAG_FIELD, true); + } + } + + void Class::setDeclaredIndexedFields(const std::vector &indexed_fields) + { + for (auto it = m_declared_field_options.begin(); it != m_declared_field_options.end();) { + it->second.clear(FieldOptions::INDEXED_FIELD); + if (it->second.value() == 0) { + it = m_declared_field_options.erase(it); + } else { + ++it; + } + } + for (const auto &field_name: indexed_fields) { + setFieldOption(m_declared_field_options, field_name, FieldOptions::INDEXED_FIELD, true); + } } bool Class::isDeclaredTagField(const char *field_name) const { - if (m_declared_tag_field_set.empty()) { + auto it = m_declared_field_options.find(field_name); + if (it == m_declared_field_options.end()) { + return false; + } + return it->second[FieldOptions::TAG_FIELD]; + } + + bool Class::isDeclaredIndexedField(const char *field_name) const + { + auto it = m_declared_field_options.find(field_name); + if (it == m_declared_field_options.end()) { return false; } - return m_declared_tag_field_set.find(field_name) != m_declared_tag_field_set.end(); + return it->second[FieldOptions::INDEXED_FIELD]; + } + + bool Class::hasDeclaredIndexedFields() const + { + for (const auto &item: m_declared_field_options) { + if (item.second[FieldOptions::INDEXED_FIELD]) { + return true; + } + } + return false; + } + + FieldFlags Class::getOwnFieldOptions(FieldID field_id) const + { + auto it = m_field_options.find(field_id.getLongIndex()); + if (it != m_field_options.end()) { + return it->second; + } + for (const auto &entry: m_index) { + const auto &member_id = entry.second.first; + for (const auto &field_info: member_id) { + if (field_info.first == field_id) { + it = m_field_options.find(member_id.primary().first.getLongIndex()); + return it == m_field_options.end() ? FieldFlags() : it->second; + } + } + } + return {}; + } + + void Class::reloadMemberOptions(FieldID field_id) const + { + auto member = m_member_cache.tryGet(field_id.getIndexAndOffset()); + if (!member) { + return; + } + auto it = m_index.find(member->m_name); + if (it == m_index.end()) { + m_member_cache.reload(field_id.getIndexAndOffset()); + return; + } + for (const auto &field_info: it->second.first) { + m_member_cache.reload(field_info.first.getIndexAndOffset()); + } + } + + FieldFlags Class::getFieldOptions(const char *field_name, const MemberID &member_id) const + { + FieldFlags result; + if (!!member_id) { + auto member = tryGetMember(member_id.primary().first); + result = member ? member->m_field_options : getOwnFieldOptions(member_id.primary().first); + } else { + auto it = m_declared_field_options.find(field_name); + if (it != m_declared_field_options.end()) { + result = it->second; + } + } + + if (m_base_class_ptr) { + result = result | m_base_class_ptr->getFieldOptions(field_name); + } + return result; } std::vector Class::getTagFieldNames() const @@ -725,6 +1052,24 @@ namespace db0::object_model return result; } + std::vector Class::getIndexedFieldNames() const + { + openIndexedFields(); + if (m_indexed_fields.isNull()) { + return {}; + } + + std::vector result; + result.reserve(m_indexed_fields.size()); + for (const auto &indexed_field: m_indexed_fields.cached()) { + auto member = tryGetMember(indexed_field.getFieldId()); + if (member) { + result.push_back(member->m_name); + } + } + return result; + } + Address Class::getSingletonAddress() const { return (*this)->m_singleton_address; } @@ -809,6 +1154,15 @@ namespace db0::object_model // NOTICE: no instance ID for the class-ref return { this->getAddress(), UniqueAddress::INSTANCE_ID_MAX }; } + + bool Class::isDescendantOf(const Class &base) const + { + auto current_ptr = this; + while (current_ptr && !(*current_ptr == base)) { + current_ptr = current_ptr->getBaseClassPtr(); + } + return current_ptr != nullptr; + } bool Class::assignDefaultTags() const { return (*this)->m_flags[ClassOptions::NO_DEFAULT_TAGS] == false; @@ -830,7 +1184,8 @@ namespace db0::object_model Class::Member Class::MemberAdapter::operator()(std::pair loc, const o_field &field) const { auto field_name = m_class.get().getFixture()->getLimitedStringPool().fetch(field.m_name); - return { FieldID(loc), m_class.get().getFidelity(loc.first), field_name }; + auto field_id = FieldID(loc); + return { field_id, m_class.get().getFidelity(loc.first), field_name, m_class.get().getOwnFieldOptions(field_id) }; } unsigned int Class::Member::getLongIndex() const { diff --git a/src/dbzero/object_model/class/Class.hpp b/src/dbzero/object_model/class/Class.hpp index e105de33..cb25f48a 100755 --- a/src/dbzero/object_model/class/Class.hpp +++ b/src/dbzero/object_model/class/Class.hpp @@ -8,7 +8,9 @@ #include #include +#include #include +#include #include #include #include @@ -60,12 +62,53 @@ namespace db0::object_model class ObjectImmutableImpl; class ObjectAnyImpl; class Class; + class Index; struct ObjectId; + enum FieldOptions: std::uint32_t + { + TAG_FIELD = 0x0001, + INDEXED_FIELD = 0x0002 + }; + + using FieldFlags = db0::FlagSet; + +} + +DECLARE_ENUM_VALUES(db0::object_model::FieldOptions, 2) + +namespace db0::object_model + +{ + // fidelity + slot index using VFidelityVector = db0::v_bvector >; using VTagFields = db0::CachedVBVector; - using Reserved0008 = VTagFields; + struct DB0_PACKED_ATTR IndexedField + { + std::uint32_t m_field_id = 0; + db0_ptr m_index_ptr; + + IndexedField() = default; + IndexedField(FieldID field_id, db0_ptr index_ptr) + : m_field_id(field_id.getLongIndex()) + , m_index_ptr(index_ptr) + { + } + + FieldID getFieldId() const + { + assert(m_field_id); + return FieldID::fromIndex((m_field_id - 1) >> 6, (m_field_id - 1) & 0x3F); + } + + bool operator==(const IndexedField &other) const + { + return m_field_id == other.m_field_id + && m_index_ptr.getAddress() == other.m_index_ptr.getAddress(); + } + }; + using VIndexedFields = db0::CachedVBVector; DB0_PACKED_BEGIN struct DB0_PACKED_ATTR o_class: public db0::o_fixed_versioned @@ -89,8 +132,8 @@ DB0_PACKED_BEGIN const std::uint32_t m_num_bases; // Version 1 fields. - db0_ptr m_reserved_0008_ptr; - db0_ptr m_reserved_0010_ptr; + db0_ptr m_tag_fields_ptr = {}; + db0_ptr m_indexed_fields_ptr = {}; o_class(RC_LimitedStringPool &, const std::string &name, std::optional module_name, const VFieldMatrix &, const VFidelityVector &, const Schema &, const char *type_id, const char *prefix_name, ClassFlags, @@ -126,9 +169,10 @@ DB0_PACKED_END FieldID m_field_id; unsigned int m_fidelity = 0; std::string m_name; + FieldFlags m_field_options; - Member(FieldID, unsigned int fidelity, const char *); - Member(FieldID, unsigned int fidelity, const std::string &); + Member(FieldID, unsigned int fidelity, const char *, FieldFlags = {}); + Member(FieldID, unsigned int fidelity, const std::string &, FieldFlags = {}); // @return full index (index + offset) as a single integer unsigned int getLongIndex() const; @@ -141,12 +185,25 @@ DB0_PACKED_END // set the model field names void setInitVars(const std::vector &init_vars); void setDeclaredTagFields(const std::vector &tag_fields); + void setDeclaredIndexedFields(const std::vector &indexed_fields); bool isDeclaredTagField(const char *field_name) const; + bool isDeclaredIndexedField(const char *field_name) const; + bool hasDeclaredIndexedFields() const; void addTagField(FieldID); void removeTagField(FieldID); const std::vector &getTagFieldIds() const; bool isTagField(FieldID) const; std::vector getTagFieldNames() const; + FieldFlags getFieldOptions(const char *field_name, const MemberID &member_id = {}) const; + void addIndexedField(FieldID); + void addIndexedField(FieldID, const db0_ptr &); + void removeIndexedField(FieldID); + std::vector getIndexedFieldIds() const; + const std::vector &getIndexedFieldRecords() const; + std::vector getIndexedFieldNames() const; + std::shared_ptr tryGetFieldIndex(FieldID) const; + std::shared_ptr tryGetFieldIndex(const char *field_name) const; + std::shared_ptr getExistingFieldIndex(FieldID) const; // Get class name in the underlying language object model std::string getName() const; @@ -275,6 +332,7 @@ DB0_PACKED_END // NOTE: this is for type compatibility only, Class objects don't have instance_id UniqueAddress getUniqueAddress() const; + bool isDescendantOf(const Class &base) const; std::uint32_t getClassRef() const; @@ -335,6 +393,8 @@ DB0_PACKED_END VFidelityVector m_fidelities; Schema m_schema; mutable VTagFields m_tag_fields; + mutable VIndexedFields m_indexed_fields; + mutable std::unordered_map > m_index_cache; std::shared_ptr m_base_class_ptr; // Field by-name index (cache) @@ -344,9 +404,8 @@ DB0_PACKED_END mutable std::vector m_unique_keys; // fields initialized on class creation (from static code analysis) std::unordered_set m_init_vars; - std::unordered_set m_declared_tag_field_set; - mutable std::unordered_set m_tag_field_id_set; - mutable bool m_has_any_tag_fields = false; + std::unordered_map m_declared_field_options; + mutable std::unordered_map m_field_options; const std::uint32_t m_uid = 0; mutable MemberCacheT m_member_cache; // runtime flags @@ -365,6 +424,12 @@ DB0_PACKED_END VTagFields &ensureTagFields(); void openTagFields() const; void onTagFieldKeyChange(const FieldID &, bool added) const; + VIndexedFields &ensureIndexedFields(); + void openIndexedFields() const; + void onIndexedFieldKeyChange(const IndexedField &, bool added) const; + FieldFlags getOwnFieldOptions(FieldID) const; + void reloadMemberOptions(FieldID) const; + std::shared_ptr tryGetOwnIndexedFieldIndex(FieldID) const; // Initialization function std::unordered_set makeInitVars(const std::vector &) const; diff --git a/src/dbzero/object_model/class/ClassFactory.cpp b/src/dbzero/object_model/class/ClassFactory.cpp index 9561b15a..61610e91 100755 --- a/src/dbzero/object_model/class/ClassFactory.cpp +++ b/src/dbzero/object_model/class/ClassFactory.cpp @@ -13,7 +13,10 @@ #include #include #include +#include +#include #include +#include #include namespace db0::object_model @@ -121,6 +124,22 @@ namespace db0::object_model return result; } + std::vector resolveDeclaredIndexedFields(const Class &type, const std::vector &indexed_fields) + { + std::vector result; + std::unordered_set seen; + for (const auto &field_name: indexed_fields) { + auto member = type.tryGetMember(field_name.c_str()); + if (member) { + auto long_index = member->m_field_id.getLongIndex(); + if (seen.insert(long_index).second) { + result.push_back(member->m_field_id); + } + } + } + return result; + } + std::unordered_set fieldIdSet(const std::vector &field_ids) { std::unordered_set result; @@ -177,18 +196,22 @@ namespace db0::object_model } template - typename TagIndex::ObjectSharedPtr tryGetMemoField(MemoT *memo_obj, const MemberLoc &member_loc) + typename TagIndex::ObjectSharedPtr tryGetMemoField(MemoT *memo_obj, const MemberLoc &member_loc, + bool *is_auto_generated = nullptr) { - return memo_obj->ext().tryGet(member_loc); + return memo_obj->ext().tryGet(member_loc, is_auto_generated); } - TagIndex::ObjectSharedPtr tryGetMemoField(TagIndex::ObjectPtr py_obj, const MemberLoc &member_loc) + TagIndex::ObjectSharedPtr tryGetMemoField(TagIndex::ObjectPtr py_obj, const MemberLoc &member_loc, + bool *is_auto_generated = nullptr) { if (db0::python::PyMemo_Check(py_obj)) { - return tryGetMemoField(reinterpret_cast(py_obj), member_loc); + return tryGetMemoField(reinterpret_cast(py_obj), member_loc, + is_auto_generated); } if (db0::python::PyMemo_Check(py_obj)) { - return tryGetMemoField(reinterpret_cast(py_obj), member_loc); + return tryGetMemoField(reinterpret_cast(py_obj), member_loc, + is_auto_generated); } return {}; } @@ -210,6 +233,34 @@ namespace db0::object_model return nullptr; } + bool hasAncestorIndexedField(const Class &type, FieldID field_id) + { + auto member = type.tryGetMember(field_id); + if (!member) { + return false; + } + auto base_type = type.getBaseClassPtr(); + while (base_type) { + if (base_type->isDeclaredIndexedField(member->m_name.c_str()) + || base_type->tryGetFieldIndex(member->m_name.c_str())) { + return true; + } + base_type = base_type->getBaseClassPtr(); + } + return false; + } + + void validateIndexedFieldAncestorOverlap(const Class &type, const std::vector &field_ids) + { + for (const auto &field_id: field_ids) { + if (hasAncestorIndexedField(type, field_id)) { + THROWF(db0::InputException) + << "Indexed field declaration overlaps an ancestor indexed field in class " + << type.getName(); + } + } + } + struct TagFieldMigrateMemberLocs { std::vector > removed; @@ -255,6 +306,40 @@ namespace db0::object_model const std::vector &m_added_ids; }; + struct IndexedFieldMigrationIndex + { + FieldID field_id; + std::shared_ptr index; + }; + + std::vector createManagedIndexes(Class &type, + const std::vector &added_ids) + { + std::vector result; + result.reserve(added_ids.size()); + auto fixture = type.getFixture(); + for (const auto &field_id: added_ids) { + auto index = std::make_shared(fixture, true); + index->setManaged(); + index->incRef(false); + result.push_back({ field_id, index }); + } + return result; + } + + void destroyManagedIndexes(const std::vector &indexes) + { + for (const auto &item: indexes) { + item.index->rollback(); + if (item.index->decRef(false)) { + auto fixture = item.index->getFixture(); + auto address = item.index->getAddress(); + item.index->destroy(); + fixture->getVObjectCache().erase(address); + } + } + } + void migrateTagFields(Class &type, const std::vector &passive_removed_ids, const std::vector &passive_added_ids, const TagFieldEdit &tag_field_edit) { @@ -343,6 +428,90 @@ namespace db0::object_model } } + void applyIndexedFieldDeclarations(Class &type, const std::vector &indexed_fields, + bool migrate, bool raise_on_mismatch) + { + if (indexed_fields.empty() && !type.hasDeclaredIndexedFields()) { + type.setDeclaredIndexedFields(indexed_fields); + return; + } + auto new_indexed_field_ids = resolveDeclaredIndexedFields(type, indexed_fields); + validateIndexedFieldAncestorOverlap(type, new_indexed_field_ids); + auto old_indexed_field_ids = type.hasDeclaredIndexedFields() + ? type.getIndexedFieldIds() + : std::vector(); + if (sameFieldIds(old_indexed_field_ids, new_indexed_field_ids)) { + type.setDeclaredIndexedFields(indexed_fields); + return; + } + if (!migrate) { + type.setDeclaredIndexedFields(type.getIndexedFieldNames()); + if (raise_on_mismatch) { + throw db0::python::MigrateException("Indexed-field declaration migration is required for class " + + type.getName()); + } + return; + } + + auto removed_ids = differenceById(old_indexed_field_ids, new_indexed_field_ids); + auto added_ids = differenceById(new_indexed_field_ids, old_indexed_field_ids); + auto new_indexes = createManagedIndexes(type, added_ids); + auto fixture = type.getFixture(); + auto &tag_index = fixture->get(); + + db0::FixtureLock lock(fixture); + tag_index.flush(); + try { + auto query = tag_index.makeIterator(type); + if (query) { + static const std::vector no_removed_ids; + TagFieldMigrateCache migrate_cache(no_removed_ids, added_ids); + ObjectIterator iterator(fixture, std::move(query), type.shared_from_this()); + for (;;) { + auto obj = iterator.next(); + if (!obj.get()) { + break; + } + auto obj_type = tryGetMemoType(obj.get()); + if (!obj_type || !obj_type->isDescendantOf(type)) { + continue; + } + auto obj_addr = ClassFactory::LangToolkit::tryGetMemoUniqueAddress(obj.get()); + if (!obj_addr) { + continue; + } + const auto &member_locs = migrate_cache.get(*obj_type); + for (std::size_t index = 0; index < member_locs.added.size(); ++index) { + const auto &member_loc = member_locs.added[index]; + if (!member_loc) { + continue; + } + bool is_auto_generated = false; + auto value = tryGetMemoField(obj.get(), *member_loc, &is_auto_generated); + if (!value || is_auto_generated) { + continue; + } + new_indexes[index].index->add(value.get(), *obj_addr); + } + } + } + for (const auto &item: new_indexes) { + item.index->flush(lock); + } + } catch (...) { + destroyManagedIndexes(new_indexes); + throw; + } + + for (const auto &item: new_indexes) { + type.addIndexedField(item.field_id, db0_ptr(*item.index)); + } + for (const auto &field_id: removed_ids) { + type.removeIndexedField(field_id); + } + type.setDeclaredIndexedFields(indexed_fields); + } + o_class_factory::o_class_factory(Memspace &memspace) : m_class_map_ptrs { VClassMap(memspace), VClassMap(memspace), VClassMap(memspace), VClassMap(memspace) } { @@ -396,6 +565,7 @@ namespace db0::object_model bool no_auto_migrate = LangToolkit::isNoAutoMigrate(*getFixture()); bool can_migrate = getFixture()->getAccessType() == AccessType::READ_WRITE && !no_auto_migrate; applyTagFieldDeclarations(*type, LangToolkit::getTagFields(lang_type), can_migrate, false); + applyIndexedFieldDeclarations(*type, LangToolkit::getIndexedFields(lang_type), can_migrate, false); // add to by-type cache it_cached = m_type_cache.insert({lang_type, type}).first; m_pending_types.push_back(lang_type); @@ -403,6 +573,8 @@ namespace db0::object_model bool no_auto_migrate = LangToolkit::isNoAutoMigrate(*getFixture()); bool can_migrate = getFixture()->getAccessType() == AccessType::READ_WRITE && !no_auto_migrate; applyTagFieldDeclarations(*it_cached->second, LangToolkit::getTagFields(lang_type), can_migrate, false); + applyIndexedFieldDeclarations(*it_cached->second, LangToolkit::getIndexedFields(lang_type), can_migrate, + false); } return it_cached->second; } @@ -443,6 +615,8 @@ namespace db0::object_model bool can_migrate = getFixture()->getAccessType() == AccessType::READ_WRITE && !no_auto_migrate; applyTagFieldDeclarations(*type, LangToolkit::getTagFields(lang_type), can_migrate, no_auto_migrate); + applyIndexedFieldDeclarations(*type, LangToolkit::getIndexedFields(lang_type), can_migrate, + no_auto_migrate); } else { auto fixture = getFixture(); if (!checkAccessType(*fixture, AccessType::READ_WRITE)) { @@ -482,9 +656,11 @@ namespace db0::object_model if (lang_type) { type->setRuntimeFlags(LangToolkit::getMemoFlags(lang_type)); type->setDeclaredTagFields(LangToolkit::getTagFields(lang_type)); + type->setDeclaredIndexedFields(LangToolkit::getIndexedFields(lang_type)); for (const auto &field_id: resolveDeclaredTagFields(*type, LangToolkit::getTagFields(lang_type))) { type->addTagField(field_id); } + applyIndexedFieldDeclarations(*type, LangToolkit::getIndexedFields(lang_type), true, false); } } @@ -538,6 +714,7 @@ namespace db0::object_model } applyTagFieldDeclarations(*type, LangToolkit::getTagFields(lang_type), true, false); + applyIndexedFieldDeclarations(*type, LangToolkit::getIndexedFields(lang_type), true, false); } std::shared_ptr ClassFactory::getType(ClassPtr ptr, std::shared_ptr type, TypeObjectPtr lang_type) const @@ -566,6 +743,8 @@ namespace db0::object_model bool can_migrate = getFixture()->getAccessType() == AccessType::READ_WRITE && !no_auto_migrate; applyTagFieldDeclarations(*it_cached->second.m_class, LangToolkit::getTagFields(lang_type), can_migrate, no_auto_migrate); + applyIndexedFieldDeclarations(*it_cached->second.m_class, LangToolkit::getIndexedFields(lang_type), + can_migrate, no_auto_migrate); } return it_cached->second.m_class; } @@ -628,6 +807,10 @@ namespace db0::object_model if (!lang_type) { lang_type = tryFindLangType(*type); } + // Register before applying language metadata; declaration migration may + // iterate objects and recursively resolve this class by pointer. + it_cached = m_ptr_cache.insert({ptr, ClassItem { type, lang_type }}).first; + m_pending_ptrs.push_back(ptr); // initialize the language model if (lang_type) { type->setInitVars(LangToolkit::getInitVars(lang_type)); @@ -636,10 +819,8 @@ namespace db0::object_model bool no_auto_migrate = LangToolkit::isNoAutoMigrate(*getFixture()); bool can_migrate = getFixture()->getAccessType() == AccessType::READ_WRITE && !no_auto_migrate; applyTagFieldDeclarations(*type, LangToolkit::getTagFields(lang_type), can_migrate, false); + applyIndexedFieldDeclarations(*type, LangToolkit::getIndexedFields(lang_type), can_migrate, false); } - // register the mapping to language specific type object - it_cached = m_ptr_cache.insert({ptr, ClassItem { type, lang_type }}).first; - m_pending_ptrs.push_back(ptr); } // register the lang type mapping if missing if (lang_type && !it_cached->second.m_lang_type) { @@ -654,6 +835,8 @@ namespace db0::object_model bool can_migrate = getFixture()->getAccessType() == AccessType::READ_WRITE && !no_auto_migrate; applyTagFieldDeclarations(*it_cached->second.m_class, LangToolkit::getTagFields(lang_type), can_migrate, no_auto_migrate); + applyIndexedFieldDeclarations(*it_cached->second.m_class, LangToolkit::getIndexedFields(lang_type), + can_migrate, no_auto_migrate); } return it_cached->second; } diff --git a/src/dbzero/object_model/index/Index.cpp b/src/dbzero/object_model/index/Index.cpp index ccb2a034..6086fc16 100755 --- a/src/dbzero/object_model/index/Index.cpp +++ b/src/dbzero/object_model/index/Index.cpp @@ -28,6 +28,14 @@ namespace db0::object_model , m_mutation_log(fixture->addMutationHandler()) { } + + Index::Index(db0::swine_ptr &fixture, bool passive, AccessFlags access_mode) + : Index(fixture, access_mode) + { + if (passive) { + setPassive(); + } + } Index::Index(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) : super_t(super_t::tag_from_address(), fixture, address, access_mode) @@ -285,6 +293,15 @@ namespace db0::object_model } } + bool Index::isSupportedKey(ObjectPtr key) + { + auto &type_manager = LangToolkit::getTypeManager(); + if (type_manager.isNull(key)) { + return true; + } + return isSupportedIndexKeyType(type_manager.getTypeId(key)); + } + void Index::add(ObjectPtr key, ObjectPtr value) { assert(hasInstance()); @@ -324,6 +341,40 @@ namespace db0::object_model } m_mutation_log->onDirty(); } + + void Index::add(ObjectPtr key, UniqueAddress value) + { + assert(hasInstance()); + assert(isPassive()); + auto &type_manager = LangToolkit::getTypeManager(); + if (type_manager.isNull(key)) { + addNull(value); + return; + } + + if (m_builder.getDataType() == IndexDataType::Auto) { + m_builder.update(type_manager.getTypeId(key)); + } + + if (!isDirty()) { + setDirty(true); + } + + switch (m_builder.getDataType()) { + case IndexDataType::Int64: + m_builder.get().add(type_manager.extractInt64(key), value); + break; + case IndexDataType::UInt64: + m_builder.get().add(type_manager.extractUInt64(key), value); + break; + default: + THROWF(db0::InputException) << "Index of type " + << static_cast(m_builder.getDataType()) + << " does not allow adding key type: " + << LangToolkit::getTypeName(key) << THROWF_END; + } + m_mutation_log->onDirty(); + } void Index::remove(ObjectPtr key, ObjectPtr value) { @@ -364,6 +415,40 @@ namespace db0::object_model } m_mutation_log->onDirty(); } + + void Index::remove(ObjectPtr key, UniqueAddress value) + { + assert(hasInstance()); + assert(isPassive()); + auto &type_manager = LangToolkit::getTypeManager(); + if (type_manager.isNull(key)) { + removeNull(value); + return; + } + + if (m_builder.getDataType() == IndexDataType::Auto) { + m_builder.update(type_manager.getTypeId(key)); + } + + if (!isDirty()) { + setDirty(true); + } + + switch (m_builder.getDataType()) { + case IndexDataType::Int64: + m_builder.get().remove(type_manager.extractInt64(key), value); + break; + case IndexDataType::UInt64: + m_builder.get().remove(type_manager.extractUInt64(key), value); + break; + default: + THROWF(db0::InputException) << "Index of type " + << static_cast(m_builder.getDataType()) + << " does not allow keys of type: " + << LangToolkit::getTypeName(key) << THROWF_END; + } + m_mutation_log->onDirty(); + } std::unique_ptr Index::range(ObjectPtr min, ObjectPtr max, bool null_first) const { @@ -400,6 +485,10 @@ namespace db0::object_model Index::sort(const ObjectIterable &iter, bool asc, bool null_first) const { assert(hasInstance()); + if (isPassive() && !iter.isNonPassiveAnchor()) { + THROWF(db0::InputException) + << "Passive index queries require at least one non-passive positive predicate" << THROWF_END; + } if (isDirty()) { FixtureLock lock(this->getFixture()); const_cast(this)->flush(lock); @@ -489,6 +578,32 @@ namespace db0::object_model } m_mutation_log->onDirty(); } + + void Index::addNull(UniqueAddress address) + { + assert(hasInstance()); + assert(isPassive()); + if (!isDirty()) { + setDirty(true); + } + + switch (m_builder.getDataType()) { + case IndexDataType::Auto: + m_builder.getAuto().addNull(address); + break; + case IndexDataType::Int64: + m_builder.get().addNull(address); + break; + case IndexDataType::UInt64: + m_builder.get().addNull(address); + break; + default: + THROWF(db0::InputException) + << "Unsupported index data type: " + << static_cast(m_builder.getDataType()) << THROWF_END; + } + m_mutation_log->onDirty(); + } // extract optional value template <> std::optional Index::extractOptionalValue(ObjectPtr value) const @@ -553,6 +668,32 @@ namespace db0::object_model } m_mutation_log->onDirty(); } + + void Index::removeNull(UniqueAddress address) + { + assert(hasInstance()); + assert(isPassive()); + if (!isDirty()) { + setDirty(true); + } + + switch (m_builder.getDataType()) { + case IndexDataType::Auto: + m_builder.getAuto().removeNull(address); + break; + case IndexDataType::Int64: + m_builder.get().removeNull(address); + break; + case IndexDataType::UInt64: + m_builder.get().removeNull(address); + break; + default: + THROWF(db0::InputException) + << "Unsupported index data type: " + << static_cast(m_builder.getDataType()) << THROWF_END; + } + m_mutation_log->onDirty(); + } void Index::moveTo(db0::swine_ptr &fixture) { @@ -606,6 +747,7 @@ namespace db0::object_model void Index::destroy() { + unregister(true); m_mutation_log = nullptr; if (!m_builder.empty() || hasRangeTree()) { this->getFixture()->detachIterators(); @@ -617,31 +759,38 @@ namespace db0::object_model auto unref_func = [&fixture](UniqueAddress objAddr) { unrefAnyMemoObject(fixture, objAddr); }; + auto maybe_unref_func = isPassive() ? std::function() : unref_func; switch ((*this)->m_data_type) { case IndexDataType::Int64: { // unreference all elements - getExistingRangeTree().forAll(unref_func); + if (maybe_unref_func) { + getExistingRangeTree().forAll(maybe_unref_func); + } getExistingRangeTree().destroy(); break; } case IndexDataType::UInt64: { // unreference all elements - getExistingRangeTree().forAll(unref_func); + if (maybe_unref_func) { + getExistingRangeTree().forAll(maybe_unref_func); + } getExistingRangeTree().destroy(); break; } case IndexDataType::Auto: { // unreference all elements - getExistingRangeTree().forAll(unref_func); + if (maybe_unref_func) { + getExistingRangeTree().forAll(maybe_unref_func); + } getExistingRangeTree().destroy(); break; } default: THROWF(db0::InputException) - << "Unsupported index data type: " + << "Unsupported index data type: " << static_cast((*this)->m_data_type); } } @@ -690,6 +839,9 @@ namespace db0::object_model auto unref_func = [&fixture](UniqueAddress objAddr) { unrefAnyMemoObject(fixture, objAddr); }; + if (isPassive()) { + return; + } switch ((*this)->m_data_type) { case IndexDataType::Int64: getExistingRangeTree().forAll(unref_func); diff --git a/src/dbzero/object_model/index/Index.hpp b/src/dbzero/object_model/index/Index.hpp index c7572a73..97fa595d 100755 --- a/src/dbzero/object_model/index/Index.hpp +++ b/src/dbzero/object_model/index/Index.hpp @@ -41,13 +41,17 @@ namespace db0::object_model // null instance constructor Index(); Index(db0::swine_ptr &, AccessFlags = {}); + Index(db0::swine_ptr &, bool passive, AccessFlags = {}); Index(db0::swine_ptr &, Address, AccessFlags = {}); Index(const Index &) = delete; ~Index(); std::size_t size() const; + static bool isSupportedKey(ObjectPtr key); void add(ObjectPtr key, ObjectPtr value); void remove(ObjectPtr key, ObjectPtr value); + void add(ObjectPtr key, UniqueAddress value); + void remove(ObjectPtr key, UniqueAddress value); /** * Sort results of a specific object iterator from the same fixture @@ -93,6 +97,22 @@ namespace db0::object_model void clearMembers(); + bool isPassive() const { + return (*this)->isPassive(); + } + + void setPassive() { + this->modify().m_flags.set(IndexOptions::Passive); + } + + bool isManaged() const { + return (*this)->isManaged(); + } + + void setManaged() { + this->modify().setManaged(); + } + protected: // the default / provisional type using DefaultT = std::int64_t; @@ -139,7 +159,7 @@ namespace db0::object_model IndexBuilder &getAuto() { if (!m_index_builder) { - m_index_builder = db0::make_shared_void >(); + m_index_builder = makeBuilder(); m_new_type = IndexDataType::Auto; } return *static_cast*>(m_index_builder.get()); @@ -148,7 +168,7 @@ namespace db0::object_model template IndexBuilder &get() { if (!m_index_builder) { - m_index_builder = db0::make_shared_void >(); + m_index_builder = makeBuilder(); m_new_type = Index::dataTypeOf(); } return *static_cast*>(m_index_builder.get()); @@ -171,7 +191,7 @@ namespace db0::object_model } if (!std::is_same_v) { - m_index_builder = db0::make_shared_void >( + m_index_builder = makeBuilder( get().releaseRemoveNullItems(), get().releaseAddNullItems(), get().releaseObjectCache() @@ -179,6 +199,22 @@ namespace db0::object_model m_new_type = Index::dataTypeOf(); } } + + template std::shared_ptr makeBuilder() const + { + return db0::make_shared_void >(m_index.isPassive()); + } + + template std::shared_ptr makeBuilder( + std::unordered_set &&remove_null_values, + std::unordered_set &&add_null_values, + std::unordered_map &&object_cache) const + { + return db0::make_shared_void >( + std::move(remove_null_values), std::move(add_null_values), std::move(object_cache), + m_index.isPassive() + ); + } }; Builder m_builder; @@ -296,6 +332,8 @@ namespace db0::object_model // adds to with a null key, compatible with all types void addNull(ObjectPtr); void removeNull(ObjectPtr); + void addNull(UniqueAddress); + void removeNull(UniqueAddress); template std::optional extractOptionalValue(ObjectPtr value) const; }; diff --git a/src/dbzero/object_model/index/IndexBuilder.hpp b/src/dbzero/object_model/index/IndexBuilder.hpp index bcee9639..a7b11fcf 100755 --- a/src/dbzero/object_model/index/IndexBuilder.hpp +++ b/src/dbzero/object_model/index/IndexBuilder.hpp @@ -24,17 +24,22 @@ namespace db0::object_model using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; using ObjectSharedExtPtr = typename LangToolkit::ObjectSharedExtPtr; - IndexBuilder(); + explicit IndexBuilder(bool passive = false); IndexBuilder(std::unordered_set &&remove_null_values, std::unordered_set &&add_null_values, - std::unordered_map &&object_cache); + std::unordered_map &&object_cache, + bool passive = false); ~IndexBuilder(); void add(KeyT key, ObjectPtr obj_ptr); void remove(KeyT key, ObjectPtr obj_ptr); + void add(KeyT key, UniqueAddress address); + void remove(KeyT key, UniqueAddress address); void addNull(ObjectPtr obj_ptr); void removeNull(ObjectPtr obj_ptr); + void addNull(UniqueAddress address); + void removeNull(UniqueAddress address); // Flush and incRef to unique added objects void flush(RangeTreeT &index); @@ -45,6 +50,7 @@ namespace db0::object_model private: typename LangToolkit::TypeManager &m_type_manager; + bool m_passive = false; // A cache of language objects held until flush/close is called // it's required to prevent unreferenced objects from being collected by GC @@ -56,17 +62,19 @@ namespace db0::object_model UniqueAddress addToCache(ObjectPtr); }; - template IndexBuilder::IndexBuilder() + template IndexBuilder::IndexBuilder(bool passive) : super_t() , m_type_manager(LangToolkit::getTypeManager()) + , m_passive(passive) { } template IndexBuilder::IndexBuilder( std::unordered_set &&remove_null_values, std::unordered_set &&add_null_values, - std::unordered_map &&object_cache) + std::unordered_map &&object_cache, bool passive) : super_t(std::move(remove_null_values), std::move(add_null_values)) , m_type_manager(LangToolkit::getTypeManager()) + , m_passive(passive) , m_object_cache(std::move(object_cache)) { } @@ -79,10 +87,20 @@ namespace db0::object_model super_t::add(key, addToCache(obj_ptr)); } - template void IndexBuilder::remove(KeyT key, ObjectPtr obj_ptr) { + template void IndexBuilder::remove(KeyT key, ObjectPtr obj_ptr) { super_t::remove(key, addToCache(obj_ptr)); } + template void IndexBuilder::add(KeyT key, UniqueAddress address) { + assert(m_passive && "Address-only index updates are only valid for passive indexes"); + super_t::add(key, address); + } + + template void IndexBuilder::remove(KeyT key, UniqueAddress address) { + assert(m_passive && "Address-only index updates are only valid for passive indexes"); + super_t::remove(key, address); + } + template void IndexBuilder::addNull(ObjectPtr obj_ptr) { super_t::addNull(addToCache(obj_ptr)); } @@ -90,9 +108,25 @@ namespace db0::object_model template void IndexBuilder::removeNull(ObjectPtr obj_ptr) { super_t::removeNull(addToCache(obj_ptr)); } + + template void IndexBuilder::addNull(UniqueAddress address) { + assert(m_passive && "Address-only index updates are only valid for passive indexes"); + super_t::addNull(address); + } + + template void IndexBuilder::removeNull(UniqueAddress address) { + assert(m_passive && "Address-only index updates are only valid for passive indexes"); + super_t::removeNull(address); + } template void IndexBuilder::flush(RangeTreeT &index) - { + { + if (m_passive) { + std::function no_op_callback = [](UniqueAddress) {}; + super_t::flush(index, &no_op_callback, &no_op_callback); + return; + } + std::function add_callback = [&](UniqueAddress address) { auto it = m_object_cache.find(address); assert(it != m_object_cache.end()); @@ -114,6 +148,9 @@ namespace db0::object_model UniqueAddress IndexBuilder::addToCache(ObjectPtr obj_ptr) { auto obj_addr = m_type_manager.extractObjectUniqueAddress(obj_ptr); + if (m_passive) { + return obj_addr; + } if (m_object_cache.find(obj_addr) == m_object_cache.end()) { m_object_cache.emplace(obj_addr, obj_ptr); } diff --git a/src/dbzero/object_model/object/Object.cpp b/src/dbzero/object_model/object/Object.cpp index 8efd568d..f23fb53a 100755 --- a/src/dbzero/object_model/object/Object.cpp +++ b/src/dbzero/object_model/object/Object.cpp @@ -3,6 +3,7 @@ #include "Object.hpp" #include +#include #include namespace db0::object_model @@ -160,6 +161,19 @@ namespace db0::object_model assert(m_type); // find already existing field index auto [member_id, is_init_var] = m_type->findField(field_name); + auto member_loc = MemberLoc { member_id, is_init_var }; + auto member = member_id ? m_type->tryGetMember(member_id.primary().first) : std::optional(); + auto managed_index = (member && member->m_field_options[FieldOptions::INDEXED_FIELD]) + ? m_type->tryGetFieldIndex(member->m_field_id) + : nullptr; + ObjectSharedPtr old_index_key; + if (managed_index && member_id) { + bool auto_generated = false; + old_index_key = tryGet(member_loc, &auto_generated); + if (auto_generated) { + old_index_key = {}; + } + } auto storage_fidelity = getStorageFidelity(storage_class); // get field ID matching the required storage fidelity FieldID field_id; @@ -173,10 +187,17 @@ namespace db0::object_model if (!member_id || !(field_id = member_id.tryGet(storage_fidelity))) { // try mutating the class first member_id = m_type->addField(field_name, storage_fidelity, m_type->isDeclaredTagField(field_name)); + member = m_type->tryGetMember(member_id.primary().first); + managed_index = (member && member->m_field_options[FieldOptions::INDEXED_FIELD]) + ? m_type->tryGetFieldIndex(member->m_field_id) + : nullptr; field_id = member_id.get(storage_fidelity); } assert(field_id && member_id); + if (managed_index && !Index::isSupportedKey(lang_value)) { + THROWF(db0::InputException) << "Unsupported index key type"; + } // NOTE: a new member inherits the parent's no-cache flag // FIXME: value should be destroyed on exception auto value = createMember( @@ -202,6 +223,16 @@ namespace db0::object_model // Either use existing slot or create a new (kv-index) addWithLoc(fixture, field_id, loc_ptr, pos, storage_fidelity, storage_class, value); } + + if (managed_index) { + auto object_addr = getUniqueAddress(); + if (!!old_index_key) { + managed_index->remove(old_index_key.get(), object_addr); + } + if (lang_value) { + managed_index->add(lang_value, object_addr); + } + } // the KV-index insert operation must be registered as the potential silent mutation // but the operation can be avoided if the object is already marked as modified @@ -238,9 +269,26 @@ namespace db0::object_model THROWF(db0::InputException) << "Attribute not found: " << field_name; } + auto member = m_type->tryGetMember(member_id.primary().first); + auto managed_index = (member && member->m_field_options[FieldOptions::INDEXED_FIELD]) + ? m_type->tryGetFieldIndex(member->m_field_id) + : nullptr; + ObjectSharedPtr old_index_key; + if (managed_index) { + bool auto_generated = false; + old_index_key = tryGet(MemberLoc { member_id, is_init_var }, &auto_generated); + if (auto_generated) { + old_index_key = {}; + } + } + // NOTE: unreference as DELETED unrefWithLoc(fixture, field_info.first, loc_ptr, pos, StorageClass::DELETED, field_info.second); + + if (managed_index && !!old_index_key) { + managed_index->remove(old_index_key.get(), getUniqueAddress()); + } // the KV-index erase operation must be registered as the potential silent mutation // but the operation can be avoided if the object is already marked as modified diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index ad6ef97c..1120bcf1 100755 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -214,6 +214,7 @@ namespace db0::object_model type.setSingletonAddress(*this); } initializer.flushTagFields(lang_object); + initializer.flushIndexedFields(this->getUniqueAddress()); initializer.close(); } @@ -271,15 +272,20 @@ namespace db0::object_model template void ObjectImplBase::setPreInit(const char *field_name, TypeId type_id, ObjectPtr obj_ptr, - bool is_tag_field) const + FieldFlags) const { assert(!this->hasInstance()); if (!LangToolkit::isValid(obj_ptr)) { auto member_id = removePreInit(field_name); - if (is_tag_field) { - auto &initializer = InitManager::instance.getInitializer(*this); + auto &initializer = InitManager::instance.getInitializer(*this); + auto &type = initializer.getClass(); + auto field_options = type.getFieldOptions(field_name, member_id); + if (field_options[FieldOptions::TAG_FIELD]) { initializer.setTagField(member_id.primary().first, obj_ptr); } + if (field_options[FieldOptions::INDEXED_FIELD]) { + initializer.clearIndexedField(member_id.primary().first); + } return; } @@ -301,6 +307,7 @@ namespace db0::object_model // use the default fidelity for the storage class member_id = type.addField(field_name, storage_fidelity, type.isDeclaredTagField(field_name)); } + auto field_options = type.getFieldOptions(field_name, member_id); if (storage_fidelity == 0) { if (member_id.hasFidelity(2)) { @@ -343,16 +350,19 @@ namespace db0::object_model auto mask = lofi_store<2>::mask(loc.second); initializer.set(loc, storage_class, value, mask); } - if (is_tag_field) { + if (field_options[FieldOptions::TAG_FIELD]) { initializer.setTagField(member_id.primary().first, obj_ptr); } + if (field_options[FieldOptions::INDEXED_FIELD]) { + initializer.setIndexedField(member_id.primary().first, obj_ptr); + } } template - void ObjectImplBase::setPreInit(const char *field_name, ObjectPtr obj_ptr, bool is_tag_field) const + void ObjectImplBase::setPreInit(const char *field_name, ObjectPtr obj_ptr, FieldFlags field_options) const { auto type_id = LangToolkit::getTypeManager().getTypeId(obj_ptr); - setPreInit(field_name, type_id, obj_ptr, is_tag_field); + setPreInit(field_name, type_id, obj_ptr, field_options); } template @@ -805,6 +815,32 @@ namespace db0::object_model } } + template + void ObjectImplBase::dropIndexedFields(Class &type) const + { + auto object_addr = this->getUniqueAddress(); + const Class *type_ptr = &type; + while (type_ptr) { + for (const auto &indexed_field: type_ptr->getIndexedFieldRecords()) { + auto field_id = indexed_field.getFieldId(); + auto index = type_ptr->tryGetFieldIndex(field_id); + if (!index) { + continue; + } + auto member = type_ptr->tryGetMember(field_id); + if (!member) { + continue; + } + bool is_auto_generated = false; + auto key = tryGet(type.findField(member->m_name.c_str()), &is_auto_generated); + if (!!key && !is_auto_generated) { + index->remove(key.get(), object_addr); + } + } + type_ptr = type_ptr->getBaseClassPtr(); + } + } + template void ObjectImplBase::dropMembers(Class &class_ref) const { @@ -858,8 +894,9 @@ namespace db0::object_model // retrieve type from the initializer type = std::const_pointer_cast(unloadType()); } - + dropTags(*type); + dropIndexedFields(*type); dropMembers(*type); // dereference associated class type->decRef(false); diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index d0d1cdcc..6ab3e08f 100755 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -5,7 +5,7 @@ #include "ObjectAnyBase.hpp" #include -#include +#include #include #include #include "o_object.hpp" @@ -103,8 +103,8 @@ namespace db0::object_model // Assign field of an uninitialized instance (assumed as a non-mutating operation) // NOTE: if lang_value is nullptr then the member is removed void setPreInit(const char *field_name, TypeId type_id, ObjectPtr lang_value, - bool is_tag_field = false) const; - void setPreInit(const char *field_name, ObjectPtr lang_value, bool is_tag_field = false) const; + FieldFlags field_options = {}) const; + void setPreInit(const char *field_name, ObjectPtr lang_value, FieldFlags field_options = {}) const; MemberID removePreInit(const char *field_name) const; ObjectSharedPtr tryGet(MemberLoc, bool *is_auto_generated = nullptr) const; @@ -187,6 +187,7 @@ namespace db0::object_model void dropMembers(db0::swine_ptr &, Class &) const; void dropMembers(Class &) const; void dropTags(Class &) const; + void dropIndexedFields(Class &) const; void unrefMember(db0::swine_ptr &, StorageClass, Value) const; void unrefMember(db0::swine_ptr &, XValue) const; diff --git a/src/dbzero/object_model/object/ObjectInitializer.cpp b/src/dbzero/object_model/object/ObjectInitializer.cpp index 924d5e28..3f6e7b4f 100755 --- a/src/dbzero/object_model/object/ObjectInitializer.cpp +++ b/src/dbzero/object_model/object/ObjectInitializer.cpp @@ -3,6 +3,7 @@ #include "ObjectInitializer.hpp" #include +#include #include #include #include @@ -23,6 +24,7 @@ namespace db0::object_model m_values.clear(); m_has_value.clear(); m_tag_fields.clear(); + m_indexed_fields.clear(); m_tag_index = nullptr; m_ref_counts = {0, 0}; m_type_initializer = {}; @@ -67,6 +69,44 @@ namespace db0::object_model } m_tag_fields.clear(); } + + void ObjectInitializer::setIndexedField(FieldID field_id, ObjectPtr value) + { + if (!Index::isSupportedKey(value)) { + THROWF(db0::InputException) << "Unsupported index key type"; + } + m_indexed_fields.push_back({ field_id, ObjectSharedPtr(value) }); + } + + void ObjectInitializer::clearIndexedField(FieldID field_id) + { + m_indexed_fields.push_back({ field_id, ObjectSharedPtr() }); + } + + void ObjectInitializer::flushIndexedFields(UniqueAddress memo_addr) + { + if (m_indexed_fields.empty()) { + return; + } + + std::stable_sort(m_indexed_fields.begin(), m_indexed_fields.end(), [](const auto &lhs, const auto &rhs) { + return lhs.m_field_id.getLongIndex() < rhs.m_field_id.getLongIndex(); + }); + + auto &type = getClass(); + for (auto field = m_indexed_fields.end(); field != m_indexed_fields.begin();) { + --field; + auto field_id = field->m_field_id; + if (!!field->m_value) { + auto index = type.getExistingFieldIndex(field_id); + index->add(field->m_value.get(), memo_addr); + } + while (field != m_indexed_fields.begin() && (field - 1)->m_field_id == field_id) { + --field; + } + } + m_indexed_fields.clear(); + } Class &ObjectInitializer::getClass() const { return *getClassPtr(); diff --git a/src/dbzero/object_model/object/ObjectInitializer.hpp b/src/dbzero/object_model/object/ObjectInitializer.hpp index 33f12786..e5e7b169 100755 --- a/src/dbzero/object_model/object/ObjectInitializer.hpp +++ b/src/dbzero/object_model/object/ObjectInitializer.hpp @@ -103,6 +103,8 @@ namespace db0::object_model public: using XValue = db0::object_model::XValue; using TypeInitializer = std::function(db0::swine_ptr &)>; + using ObjectPtr = LangConfig::ObjectPtr; + using ObjectSharedPtr = LangConfig::ObjectSharedPtr; virtual ~ObjectInitializer() = default; @@ -158,6 +160,9 @@ namespace db0::object_model virtual bool remove(std::pair loc, std::uint64_t mask = 0); void setTagField(FieldID field_id, ObjectPtr value); void flushTagFields(ObjectPtr memo_ptr); + void setIndexedField(FieldID field_id, ObjectPtr value); + void clearIndexedField(FieldID field_id); + void flushIndexedFields(UniqueAddress memo_addr); // Allows migrating initialization to other fixture (only for empty ObjectInitializer) // @return false if operation failed (exception not thrown) @@ -231,6 +236,12 @@ namespace db0::object_model mutable SparseBoolMatrix m_has_value; std::vector m_tag_fields; TagIndex *m_tag_index = nullptr; + struct IndexedFieldValue + { + FieldID m_field_id; + ObjectSharedPtr m_value; + }; + std::vector m_indexed_fields; std::pair m_ref_counts = {0, 0}; mutable db0::swine_ptr m_fixture; mutable TypeInitializer m_type_initializer; diff --git a/src/dbzero/object_model/tags/ObjectIterable.cpp b/src/dbzero/object_model/tags/ObjectIterable.cpp index dadbc5cc..5fa30d0f 100755 --- a/src/dbzero/object_model/tags/ObjectIterable.cpp +++ b/src/dbzero/object_model/tags/ObjectIterable.cpp @@ -38,7 +38,7 @@ namespace db0::object_model ObjectIterable::ObjectIterable(db0::swine_ptr fixture, std::unique_ptr &&ft_query_iterator, std::shared_ptr type, TypeObjectPtr lang_type, std::vector > &&query_observers, - const std::vector &filters) + const std::vector &filters, QueryPlanning query_planning) : m_fixture(fixture) , m_class_factory(getClassFactory(*fixture)) , m_query_iterator(validated(std::move(ft_query_iterator))) @@ -47,12 +47,13 @@ namespace db0::object_model , m_type(type) , m_lang_type(lang_type) , m_access_mode(getAccessMode(type)) + , m_query_planning(query_planning) { } ObjectIterable::ObjectIterable(db0::swine_ptr fixture, std::unique_ptr &&sorted_iterator, std::shared_ptr type, TypeObjectPtr lang_type, std::vector > &&query_observers, - const std::vector &filters) + const std::vector &filters, QueryPlanning query_planning) : m_fixture(fixture) , m_class_factory(getClassFactory(*fixture)) , m_sorted_iterator(validated(std::move(sorted_iterator))) @@ -61,12 +62,13 @@ namespace db0::object_model , m_type(type) , m_lang_type(lang_type) , m_access_mode(getAccessMode(type)) + , m_query_planning(query_planning) { } ObjectIterable::ObjectIterable(db0::swine_ptr fixture, std::shared_ptr factory, std::shared_ptr type, TypeObjectPtr lang_type, std::vector > &&query_observers, - const std::vector &filters) + const std::vector &filters, QueryPlanning query_planning) : m_fixture(fixture) , m_class_factory(getClassFactory(*fixture)) , m_factory(factory) @@ -75,6 +77,7 @@ namespace db0::object_model , m_type(type) , m_lang_type(lang_type) , m_access_mode(getAccessMode(type)) + , m_query_planning(query_planning) { } @@ -82,7 +85,7 @@ namespace db0::object_model std::unique_ptr &&ft_query_iterator, std::unique_ptr &&sorted_iterator, std::shared_ptr factory, std::vector > &&query_observers, std::vector &&filters, std::shared_ptr type, TypeObjectPtr lang_type, - const SliceDef &slice_def, AccessFlags access_mode) + const SliceDef &slice_def, AccessFlags access_mode, QueryPlanning query_planning) : m_fixture(fixture) , m_class_factory(class_factory) , m_query_iterator(std::move(ft_query_iterator)) @@ -94,6 +97,7 @@ namespace db0::object_model , m_lang_type(lang_type) , m_slice_def(slice_def) , m_access_mode(access_mode) + , m_query_planning(query_planning) { } @@ -106,6 +110,7 @@ namespace db0::object_model , m_lang_type(other.m_lang_type) , m_slice_def(other.m_slice_def) , m_access_mode(other.m_access_mode) + , m_query_planning(other.m_query_planning) { m_filters.insert(m_filters.end(), filters.begin(), filters.end()); @@ -127,6 +132,7 @@ namespace db0::object_model , m_lang_type(other.m_lang_type) , m_slice_def(other.m_slice_def.combineWith(slice_def)) , m_access_mode(other.m_access_mode) + , m_query_planning(other.m_query_planning) { std::unique_ptr query_iterator; std::unique_ptr sorted_iterator; @@ -150,6 +156,7 @@ namespace db0::object_model , m_lang_type(other.m_lang_type) , m_slice_def(other.m_slice_def) , m_access_mode(other.m_access_mode) + , m_query_planning(other.m_query_planning) { m_filters.insert(m_filters.end(), filters.begin(), filters.end()); } @@ -166,6 +173,7 @@ namespace db0::object_model , m_lang_type(other.m_lang_type) , m_slice_def(other.m_slice_def) , m_access_mode(other.m_access_mode) + , m_query_planning(other.m_query_planning) { m_filters.insert(m_filters.end(), filters.begin(), filters.end()); } @@ -433,6 +441,14 @@ namespace db0::object_model return true; } + void ObjectIterable::requirePassiveAnchor() const + { + if (m_query_planning.m_is_passive && !m_query_planning.m_is_anchor) { + THROWF(db0::InputException) + << "Passive index queries require at least one non-passive positive predicate" << THROWF_END; + } + } + AccessFlags ObjectIterable::getAccessMode(std::shared_ptr type) const { if (type) { diff --git a/src/dbzero/object_model/tags/ObjectIterable.hpp b/src/dbzero/object_model/tags/ObjectIterable.hpp index 9ca85fff..5c563c93 100755 --- a/src/dbzero/object_model/tags/ObjectIterable.hpp +++ b/src/dbzero/object_model/tags/ObjectIterable.hpp @@ -48,22 +48,36 @@ namespace db0::object_model using BaseIterator = db0::FT_IteratorBase; using FilterFunc = std::function; + struct QueryPlanning + { + // True when this iterable is backed by, or includes, a passive index scan. + bool m_is_passive = false; + // True if this iterable is, or already includes, a positive non-passive predicate. + bool m_is_anchor = false; + + QueryPlanning(bool is_passive = false, bool is_anchor = false) + : m_is_passive(is_passive) + , m_is_anchor(is_anchor) + { + } + }; + ObjectIterable(ObjectIterable &&) = default; // Construct from a full-text query iterator ObjectIterable(db0::swine_ptr, std::unique_ptr &&, std::shared_ptr = nullptr, TypeObjectPtr lang_type = nullptr, std::vector > && = {}, - const std::vector & = {}); + const std::vector & = {}, QueryPlanning query_planning = QueryPlanning { false, true }); // Construct from a sorted iterator ObjectIterable(db0::swine_ptr, std::unique_ptr &&, std::shared_ptr = nullptr, TypeObjectPtr lang_type = nullptr, std::vector > && = {}, - const std::vector & = {}); + const std::vector & = {}, QueryPlanning query_planning = QueryPlanning { false, true }); // Construct from IteratorFactory (specialized on first use) ObjectIterable(db0::swine_ptr, std::shared_ptr factory, std::shared_ptr = nullptr, TypeObjectPtr lang_type = nullptr, std::vector > && = {}, - const std::vector & = {}); + const std::vector & = {}, QueryPlanning query_planning = QueryPlanning { false, true }); // Construct with additional filters ObjectIterable(const ObjectIterable &, const std::vector &); @@ -134,6 +148,16 @@ namespace db0::object_model bool empty() const; + bool mayReadPassiveEntries() const { + return m_query_planning.m_is_passive; + } + + bool isNonPassiveAnchor() const { + return m_query_planning.m_is_anchor; + } + + void requirePassiveAnchor() const; + protected: mutable db0::weak_swine_ptr m_fixture; const ClassFactory &m_class_factory; @@ -148,11 +172,12 @@ namespace db0::object_model mutable ObjectSharedPtr m_lang_context; // object access mode (e.g. no_cache) const AccessFlags m_access_mode; + const QueryPlanning m_query_planning = {}; // iter constructor ObjectIterable(db0::swine_ptr, const ClassFactory &, std::unique_ptr &&, std::unique_ptr &&, std::shared_ptr, std::vector > &&, std::vector &&filters, std::shared_ptr, TypeObjectPtr lang_type, const SliceDef & = {}, - AccessFlags access_mode = {}); + AccessFlags access_mode = {}, QueryPlanning query_planning = QueryPlanning()); // get the base iterator, possibly initialized from the factory const BaseIterator &getBaseIterator(std::unique_ptr &) const; diff --git a/src/dbzero/object_model/tags/TagIndex.cpp b/src/dbzero/object_model/tags/TagIndex.cpp index f0142686..4dcea6e8 100755 --- a/src/dbzero/object_model/tags/TagIndex.cpp +++ b/src/dbzero/object_model/tags/TagIndex.cpp @@ -773,7 +773,7 @@ namespace db0::object_model std::size_t offset = 0; bool result = !no_result; bool has_passive_predicate = false; - bool has_positive_anchor = type || !native_args.empty(); + bool has_positive_anchor = type != nullptr; // apply type filter if provided (unless type is a MemoBase) if (type) { result &= m_base_index_short.addIterator(factory, ShortTagT::fromAddress(type->getAddress())); @@ -788,6 +788,9 @@ namespace db0::object_model } for (auto *native_arg: native_args) { assert(native_arg); + if (native_arg->mayReadPassiveEntries()) { + has_passive_predicate = true; + } result &= addIterator(*native_arg, factory, neg_iterators, observers, &has_positive_anchor); } if (has_passive_predicate && !has_positive_anchor) { @@ -824,7 +827,7 @@ namespace db0::object_model return false; } if (has_positive_anchor) { - *has_positive_anchor = true; + *has_positive_anchor = *has_positive_anchor || obj_iter.isNonPassiveAnchor(); } factory.add(std::move(ft_query)); return true; From 06d4398824bd34bdb959e46a1fdbbe90a59a4cad Mon Sep 17 00:00:00 2001 From: Wojtek Date: Fri, 17 Jul 2026 12:24:15 +0200 Subject: [PATCH 2/4] version update --- dbzero/setup.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbzero/setup.py b/dbzero/setup.py index e609ad91..690f67bd 100755 --- a/dbzero/setup.py +++ b/dbzero/setup.py @@ -10,7 +10,7 @@ setup( name='dbzero', - version='0.5.2', + version='0.6.0', description='DBZero community edition', packages=['dbzero'], python_requires='>=3.9', diff --git a/pyproject.toml b/pyproject.toml index f440af47..969594ab 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = ['meson-python'] [project] name = 'dbzero' -version = '0.5.2' +version = '0.6.0' description = 'A state management system for Python 3.x that unifies your applications business logic, data persistence, and caching into a single, efficient layer.' readme = 'README.md' requires-python = '>=3.9' From acf2e414bdc1be2f90b7c8b8e0300f7f7c3543cc Mon Sep 17 00:00:00 2001 From: Wojtek Date: Fri, 17 Jul 2026 12:41:00 +0200 Subject: [PATCH 3/4] py 3.13 fixes --- dbzero/dbzero/memo.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbzero/dbzero/memo.py b/dbzero/dbzero/memo.py index 641a14f7..2c7be7e4 100755 --- a/dbzero/dbzero/memo.py +++ b/dbzero/dbzero/memo.py @@ -283,6 +283,11 @@ def _is_self_load_instruction(inst): if inst.opname == "LOAD_FAST" and inst.arg == 0: # Traditional single load of first argument (self) return True + elif inst.opname == "LOAD_FAST_LOAD_FAST": + # Python 3.13+ dual local load + # argval is a tuple; self is the second value loaded for STORE_ATTR. + if isinstance(inst.argval, tuple) and len(inst.argval) == 2: + return inst.argval[1] == 'self' elif inst.opname == "LOAD_FAST_BORROW" and inst.arg == 0: # Python 3.14+ borrowed reference load of first argument (self) return True From 9c923f169a7cdf626d4b46e7790b27c93e832662 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Fri, 17 Jul 2026 13:37:28 +0200 Subject: [PATCH 4/4] VCachedBVector issue fix --- python_tests/test_indexed_fields.py | 39 +++++++++++++++++++ .../collections/vector/CachedVBVector.hpp | 4 +- .../core/collections/vector/v_bvector.hpp | 34 +++++++++++++--- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/python_tests/test_indexed_fields.py b/python_tests/test_indexed_fields.py index 627941e4..28bea5cc 100644 --- a/python_tests/test_indexed_fields.py +++ b/python_tests/test_indexed_fields.py @@ -275,6 +275,45 @@ def __init__(self, index, items): operation() +def test_indexed_field_metadata_reopens_with_no_auto_migrate(tmp_path): + def declare_task(): + @db0.indexed_fields("priority") + @db0.memo(id="dbzero-software/dbzero/tests/indexed-fields-no-auto-reopen-task") + class IndexedFieldsNoAutoReopenTask: + def __init__(self, priority): + self.priority = priority + + return IndexedFieldsNoAutoReopenTask + + def declare_root(): + @db0.memo(singleton=True, id="dbzero-software/dbzero/tests/indexed-fields-no-auto-reopen-root") + class IndexedFieldsNoAutoReopenRoot: + def __init__(self): + self.items = [] + + return IndexedFieldsNoAutoReopenRoot + + db0.init(str(tmp_path), no_auto_migrate=True) + db0.open("indexed-field-no-auto-reopen") + try: + IndexedFieldsNoAutoReopenTask = declare_task() + IndexedFieldsNoAutoReopenRoot = declare_root() + IndexedFieldsNoAutoReopenRoot().items.append(IndexedFieldsNoAutoReopenTask(7)) + db0.commit() + db0.close() + + db0.init(str(tmp_path), no_auto_migrate=True) + db0.open("indexed-field-no-auto-reopen") + IndexedFieldsNoAutoReopenTask = declare_task() + declare_root() + + index = db0.index_of(IndexedFieldsNoAutoReopenTask, "priority") + assert len(index) == 1 + assert _get_indexed_fields(IndexedFieldsNoAutoReopenTask) == ("priority",) + finally: + db0.close() + + def test_indexed_field_rejects_unsupported_string_keys(db0_fixture): @db0.indexed_fields("code") @db0.memo diff --git a/src/dbzero/core/collections/vector/CachedVBVector.hpp b/src/dbzero/core/collections/vector/CachedVBVector.hpp index 9cca69e9..2aaf7c6d 100644 --- a/src/dbzero/core/collections/vector/CachedVBVector.hpp +++ b/src/dbzero/core/collections/vector/CachedVBVector.hpp @@ -86,7 +86,7 @@ DB0_PACKED_END removed_items = m_cache; } super_t::init(memspace, access_mode); - m_vector.initUnique(memspace, access_mode); + m_vector.init(memspace, flags, access_mode); m_cache.clear(); auto &self = this->modify(); self.m_vector_ptr = m_vector; @@ -101,7 +101,7 @@ DB0_PACKED_END void init(mptr ptr, AccessFlags access_mode = {}) { super_t::operator=(super_t(ptr, access_mode)); - m_vector = vector_t(this->getMemspace().myPtr((*this)->m_vector_ptr.getAddress()), access_mode); + m_vector.init(this->getMemspace().myPtr((*this)->m_vector_ptr.getAddress()), access_mode); refreshCacheFromVector(); m_detached = false; } diff --git a/src/dbzero/core/collections/vector/v_bvector.hpp b/src/dbzero/core/collections/vector/v_bvector.hpp index 9d3aaaae..d6274a8b 100755 --- a/src/dbzero/core/collections/vector/v_bvector.hpp +++ b/src/dbzero/core/collections/vector/v_bvector.hpp @@ -113,6 +113,30 @@ DB0_PACKED_END , m_pb_mask(ptr_container::mask((*this)->m_page_size)) { } + + void init(Memspace &mem, BVectorFlags flags = {}, AccessFlags access_mode = {}) + { + super_t::init(mem, mem.getPageSize(), flags, access_mode); + this->m_db_shift = data_container::shift(mem.getPageSize()); + this->m_db_mask = data_container::mask(mem.getPageSize()); + this->m_pb_shift = ptr_container::shift(mem.getPageSize()); + this->m_pb_mask = ptr_container::mask(mem.getPageSize()); + this->m_pb_cache.clear(); + this->m_last_block_key = {0, 0}; + this->m_last_block = nullptr; + } + + void init(mptr ptr, AccessFlags access_mode = {}) + { + super_t::operator=(super_t(ptr, access_mode)); + this->m_db_shift = data_container::shift((*this)->m_page_size); + this->m_db_mask = data_container::mask((*this)->m_page_size); + this->m_pb_shift = ptr_container::shift((*this)->m_page_size); + this->m_pb_mask = ptr_container::mask((*this)->m_page_size); + this->m_pb_cache.clear(); + this->m_last_block_key = {0, 0}; + this->m_last_block = nullptr; + } v_bvector(const v_bvector &&other) : super_t(std::move(other)) @@ -125,16 +149,16 @@ DB0_PACKED_END void operator=(v_bvector &&other) { - assert(this->m_db_shift == other.m_db_shift); - assert(this->m_db_mask == other.m_db_mask); - assert(this->m_pb_shift == other.m_pb_shift); - assert(this->m_pb_mask == other.m_pb_mask); - // clean local cached objects first this->m_pb_cache.clear(); this->m_last_block_key = {0, 0}; this->m_last_block = nullptr; super_t::operator=(std::move(other)); + this->m_db_shift = other.m_db_shift; + this->m_db_mask = other.m_db_mask; + this->m_pb_shift = other.m_pb_shift; + this->m_pb_mask = other.m_pb_mask; + this->m_b_class = other.m_b_class; } // Construct populated with values from a specific sequence