Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@dpdani
dpdani committed Sep 15, 2024
1 parent 433c7d9 commit 5ff43a2
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 57 deletions.
6 changes: 3 additions & 3 deletions src/cereggii/_cereggii.pyi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,7 @@ class AtomicDict:
objects cannot be used as keys nor values.
"""

def __init__(
self, iterable: dict | None = None, /, *, min_size: int | None = None, buffer_size: int = 4, **kwargs
):
def __init__(self, iterable: dict | None = None, /, *, min_size: int | None = None, buffer_size: int = 4, **kwargs):
"""
:param iterable: An iterable to initialize this dictionary with. For now,
only `dict` can be supplied to this parameter.
Expand Down Expand Up @@ -374,12 +372,14 @@ class AtomicDict:
def _debug(self) -> dict:
"""
Provide some debugging information.
For internal usage only.
This method is subject to change without a deprecation notice.
"""
def _rehash(self, o: object) -> int:
"""
Rehash object `o` with `AtomicDict`'s internal hashing function.
For internal usage only.
This method is subject to change without a deprecation notice.
"""
Expand Down
2 changes: 1 addition & 1 deletion tests/atomic_dict_hashing_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
keys_for_hash_for_log_size[log_size][pos] = []

for _ in range((1 << log_size) * 32):
h = d.rehash(_) >> (64 - log_size)
h = d._rehash(_) >> (64 - log_size)
if h == pos:
keys_for_hash_for_log_size[log_size][pos].append(_)

Expand Down
106 changes: 53 additions & 53 deletions tests/test_atomic_dict.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def test_weird_init():

def test_debug():
d = AtomicDict({"key": "value"}, min_size=64, iterable={1: 0})
dbg = d.debug()
dbg = d._debug()
assert type(dbg) is dict # noqa: E721
assert "meta" in dbg
assert "index" in dbg
Expand All @@ -56,7 +56,7 @@ def test_debug():
previous = None
while (this := gc.collect()) != previous:
previous = this
d.debug()
d._debug()
previous = None
while (this := gc.collect()) != previous:
previous = this
Expand Down Expand Up @@ -155,7 +155,7 @@ def test_setitem_distance_1_insert():
d[pos_0_again] = 42
assert d[pos_0] == 1
assert d[pos_0_again] == 42
debug = d.debug()
debug = d._debug()
assert debug["index"][1] == 10
d = AtomicDict()
d[pos_0] = 1
Expand All @@ -167,7 +167,7 @@ def test_setitem_distance_1_insert():
assert d[pos_0] == 1
assert d[pos_1] == 2
assert d[pos_0_again] == 3
debug = d.debug()
debug = d._debug()
assert debug["index"][0] == 7
assert debug["index"][1] == 14
assert debug["index"][2] == 10
Expand All @@ -183,30 +183,30 @@ def test_insert_non_compact():

def test_full_dict():
d = AtomicDict({k: None for k in range(63)})
assert len(d.debug()["index"]) == 128
assert len(d._debug()["index"]) == 128

d = AtomicDict(min_size=64)
for k in range(62):
d[k] = None
assert len(d.debug()["index"]) == 64
assert len(d._debug()["index"]) == 64

d = AtomicDict({k: None for k in range((1 << 10) - 1)})
assert len(d.debug()["index"]) == 1 << 11
assert len(d._debug()["index"]) == 1 << 11
d = AtomicDict(min_size=1 << 10)
for k in range((1 << 10) - 2):
d[k] = None
assert len(d.debug()["index"]) == 1 << 11
assert len(d._debug()["index"]) == 1 << 11


@pytest.mark.skip()
def test_full_dict_32():
# this test is slow (allocates a lot of memory)
d = AtomicDict({k: None for k in range((1 << 25) - 1)})
assert len(d.debug()["index"]) == 1 << 25
assert len(d._debug()["index"]) == 1 << 25
d = AtomicDict(min_size=1 << 25)
for k in range((1 << 25) - 2):
d[k] = None
assert len(d.debug()["index"]) == 1 << 26
assert len(d._debug()["index"]) == 1 << 26


def test_dealloc():
Expand Down Expand Up @@ -271,11 +271,11 @@ def test_delete():
keys = keys_for_hash_for_log_size[6] # noqa: F841

# d = AtomicDict({keys[_][0]: None for _ in range(15)})
# assert d.debug()["blocks"][0]["entries"][14] # exists
# assert d._debug()["blocks"][0]["entries"][14] # exists
# del d[keys[14][0]]
# with raises(KeyError):
# d[keys[14][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][14] == debug["meta"]["tombstone"]
# with raises(IndexError):
# debug["blocks"][0]["entries"][14]
Expand All @@ -284,75 +284,75 @@ def test_delete():
# del d[keys[14][0]]
# with raises(KeyError):
# d[keys[14][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][14] == debug["meta"]["tombstone"]

# d = AtomicDict({keys[_][0]: None for _ in range(15)})
# d[keys[14][1]] = None
# del d[keys[14][0]]
# with raises(KeyError):
# d[keys[14][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][15] == debug["meta"]["tombstone"]

# d = AtomicDict({keys[_][0]: None for _ in range(15)})
# del d[keys[7][0]]
# with raises(KeyError):
# d[keys[7][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][7] == debug["meta"]["tombstone"]

# d = AtomicDict({keys[_][0]: None for _ in range(8)})
# for _ in range(7, 7 + 7):
# d[keys[_][1]] = None
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][14] != 0
# del d[keys[7][0]]
# with raises(KeyError):
# d[keys[7][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][14] == debug["meta"]["tombstone"]

# d = AtomicDict({keys[_][0]: None for _ in range(16)})
# del d[keys[15][0]]
# with raises(KeyError):
# d[keys[15][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][15] == debug["meta"]["tombstone"]

# d = AtomicDict({keys[_][0]: None for _ in range(17)})
# del d[keys[15][0]]
# with raises(KeyError):
# d[keys[15][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][15] == debug["meta"]["tombstone"]

# d = AtomicDict({keys[16][0]: None})
# del d[keys[16][0]]
# with raises(KeyError):
# d[keys[16][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][16] == debug["meta"]["tombstone"]

# d = AtomicDict({keys[16][0]: None, keys[17][0]: None})
# del d[keys[16][0]]
# with raises(KeyError):
# d[keys[16][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][16] == debug["meta"]["tombstone"]

# d = AtomicDict({keys[15][0]: None, keys[16][0]: None})
# del d[keys[16][0]]
# with raises(KeyError):
# d[keys[16][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][16] == debug["meta"]["tombstone"]

# d = AtomicDict({keys[15][0]: None, keys[16][0]: None, keys[17][0]: None})
# del d[keys[16][0]]
# with raises(KeyError):
# d[keys[16][0]]
# debug = d.debug()
# debug = d._debug()
# assert debug["index"][16] == debug["meta"]["tombstone"]


Expand All @@ -362,11 +362,11 @@ def test_delete_with_swap():

keys = keys_for_hash_for_log_size[8]
d = AtomicDict({keys[_][0]: None for _ in range(64)} | {keys[_][1]: None for _ in range(64)})
assert d.debug()["meta"]["log_size"] == 9
assert d._debug()["meta"]["log_size"] == 9
del d[keys[0][1]]
with raises(KeyError):
del d[keys[0][1]]
debug = d.debug()
debug = d._debug()
assert debug["index"][1] == debug["meta"]["tombstone"]
assert debug["index"][16] >> (debug["meta"]["node_size"] - debug["meta"]["log_size"]) == 65
assert debug["blocks"][1]["entries"][1] == (65, 128, keys[8][0], keys[8][0], None)
Expand Down Expand Up @@ -438,11 +438,11 @@ def test_memory_leak():
def test_grow():
keys = keys_for_hash_for_log_size[6]
d = AtomicDict({keys[0][0]: None, keys[1][0]: None, keys[0][1]: None, keys[1][1]: None})
assert d.debug()["meta"]["log_size"] == 6
assert len(list(filter(lambda _: _ != 0, Counter(d.debug()["index"]).keys()))) == len({0, 1, 64, 65})
assert d._debug()["meta"]["log_size"] == 6
assert len(list(filter(lambda _: _ != 0, Counter(d._debug()["index"]).keys()))) == len({0, 1, 64, 65})
d[keys[0][2]] = None
assert d.debug()["meta"]["log_size"] == 7
nodes = Counter(d.debug()["index"])
assert d._debug()["meta"]["log_size"] == 7
nodes = Counter(d._debug()["index"])
assert len(list(filter(lambda _: _ != 0, nodes))) == len({0, 1, 64, 65, 128})
for _ in nodes:
if _ != 0:
Expand All @@ -451,9 +451,9 @@ def test_grow():
assert d[_] is None

d = AtomicDict({keys[_][0]: None for _ in range(63)})
assert d.debug()["meta"]["log_size"] == 7
assert d._debug()["meta"]["log_size"] == 7
d[keys[0][1]] = None
assert d.debug()["meta"]["log_size"] == 7
assert d._debug()["meta"]["log_size"] == 7
for _ in [*[keys[k][0] for k in range(63)], keys[0][1]]:
assert d[_] is None

Expand All @@ -470,14 +470,14 @@ def test_compact():
)
for _ in range(20):
d[keys[0][_]] = None
assert len(list(filter(lambda _: _ != 0, Counter(d.debug()["index"]).keys()))) == len(
assert len(list(filter(lambda _: _ != 0, Counter(d._debug()["index"]).keys()))) == len(
{keys[0][0], keys[1][0], keys[0][1], keys[1][1], *[keys[0][i] for i in range(_ + 1)]}
), _
for _ in [keys[0][0], keys[1][0], keys[0][1], keys[1][1]]:
assert d[_] is None
for _ in range(20):
assert d[keys[0][_]] is None
debug_before = d.debug()
debug_before = d._debug()
assert len(list(filter(lambda _: _ != 0, Counter(debug_before["index"]).keys()))) == len(
{keys[0][0], keys[1][0], keys[0][1], keys[1][1], *[keys[0][_] for _ in range(20)]}
)
Expand All @@ -499,7 +499,7 @@ def test_compact():
node_of_8192 = node_of_8192[0]
assert node_of_8192 & debug_before["meta"]["distance_mask"] == 0
d.compact()
debug_after = d.debug()
debug_after = d._debug()
assert len(list(filter(lambda _: _ != 0, debug_after["index"]))) == non_empty_nodes
assert len(list(filter(lambda _: _ != 0, Counter(debug_before["index"]).keys()))) == len(
{keys[0][0], keys[1][0], keys[0][1], keys[1][1], *[keys[0][_] for _ in range(20)]}
Expand All @@ -523,82 +523,82 @@ def test_compact():
d = AtomicDict({keys[0][0]: None, keys[1][0]: None})
for _ in range(20):
d[keys[0][_]] = None
assert d.debug()["meta"]["log_size"] == 7
assert d._debug()["meta"]["log_size"] == 7
d.compact()
for _ in range(20):
assert d[keys[0][_]] is None
assert d.debug()["meta"]["log_size"] == 9
assert d._debug()["meta"]["log_size"] == 9

d = AtomicDict({}, min_size=2**16)
assert len(d.debug()["index"]) == 2**16
assert len(d._debug()["index"]) == 2**16
d.compact()
assert len(d.debug()["index"]) == 2**16
assert len(d._debug()["index"]) == 2**16


def test_grow_then_shrink():
d = AtomicDict()
assert d.debug()["meta"]["log_size"] == 6
assert d._debug()["meta"]["log_size"] == 6

for _ in range(2**10):
debug = d.debug()
debug = d._debug()
assert len(Counter(debug["index"]).keys()) == _ + 1, debug["meta"]["log_size"]
d[_] = None
assert d.debug()["meta"]["log_size"] == 11
assert d._debug()["meta"]["log_size"] == 11

for _ in range(2**10):
del d[_]
# assert d.debug()["meta"]["log_size"] == 7 # cannot shrink back to 6
# assert d._debug()["meta"]["log_size"] == 7 # cannot shrink back to 6

# debug = d.debug()
# debug = d._debug()
# empty = 0
# tombstone = debug["meta"]["tombstone"]
# assert len(Counter(debug["index"]).keys()) == len({empty, tombstone}), debug["meta"]["log_size"]

for _ in range(2**20, 2**20 + 2**14):
d[_] = None

assert d.debug()["meta"]["log_size"] == 15
# assert len(Counter(d.debug()["index"]).keys()) == 2**14 + 1
assert d._debug()["meta"]["log_size"] == 15
# assert len(Counter(d._debug()["index"]).keys()) == 2**14 + 1

for _ in range(2**20, 2**20 + 2**14):
del d[_]
# assert d.debug()["meta"]["log_size"] == 7
# assert d._debug()["meta"]["log_size"] == 7


@pytest.mark.skip()
def test_large_grow_then_shrink():
d = AtomicDict()
assert d.debug()["meta"]["log_size"] == 6
assert d._debug()["meta"]["log_size"] == 6

for _ in range(2**25):
d[_] = None
assert d.debug()["meta"]["log_size"] == 26
assert d._debug()["meta"]["log_size"] == 26

for _ in range(2**25):
del d[_]
assert d.debug()["meta"]["log_size"] == 7
assert d._debug()["meta"]["log_size"] == 7


@pytest.mark.skip()
def test_large_pre_allocated_grow_then_shrink():
d = AtomicDict(min_size=2**26)
assert d.debug()["meta"]["log_size"] == 26
assert d._debug()["meta"]["log_size"] == 26

for _ in range(2**25):
d[_] = None
assert d.debug()["meta"]["log_size"] == 26
assert d._debug()["meta"]["log_size"] == 26

for _ in range(2**25):
del d[_]
assert d.debug()["meta"]["log_size"] == 26
assert d._debug()["meta"]["log_size"] == 26


def test_dont_implode():
d = AtomicDict({1: None, 10: None})
del d[1]
d[2] = None
assert d[2] is None
assert d.debug()["meta"]["log_size"] == 6
assert d._debug()["meta"]["log_size"] == 6


def test_len_bounds():
Expand Down

0 comments on commit 5ff43a2

Please sign in to comment.