Skip to content

Commit cbccd01

Browse files
mkittimkitti
committed
Add initial prototype
1 parent 84e6632 commit cbccd01

File tree

7 files changed

+196
-1
lines changed

7 files changed

+196
-1
lines changed

‎CondaPkg.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[deps]
2+
tensorstore = ""

‎Project.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,13 @@ uuid = "c6b2ce88-6a03-4336-b453-a8c964d638dc"
33
authors = ["Mark Kittisopikul <kittisopikulm@janelia.hhmi.org> and contributors"]
44
version = "1.0.0-DEV"
55

6+
[deps]
7+
CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
8+
PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
9+
610
[compat]
11+
CondaPkg = "0.2.29"
12+
PythonCall = "0.9.25"
713
julia = "1.10"
814

915
[extras]

‎README.md‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,127 @@
11
# PyTensorStore
22

33
[![Build Status](https://github.com/mkitti/PyTensorStore.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/mkitti/PyTensorStore.jl/actions/workflows/CI.yml?query=branch%3Amain)
4+
5+
PyTensorStore.jl provides a wrapper around the Python package `tensorstore`. A future TensorStore.jl may wrap the C++ API directly.
6+
7+
This package is being primarily developed to test Zarr.jl.
8+
9+
## Usage
10+
11+
```julia-repl
12+
julia> using PyTensorStore
13+
Precompiling PyTensorStore...
14+
1 dependency successfully precompiled in 2 seconds. 50 already precompiled.
15+
16+
julia> d = Dict(
17+
"driver" => "n5",
18+
"kvstore" => Dict(
19+
"driver" => "file",
20+
"path" => "tmp/dataset/",
21+
),
22+
"metadata" => Dict(
23+
"compression" => Dict(
24+
"type" => "gzip"
25+
),
26+
"dataType" => "uint32",
27+
"dimensions" => [1000, 20000],
28+
"blockSize" => [100, 100],
29+
),
30+
"create" => true,
31+
"delete_existing" => true
32+
)
33+
Dict{String, Any} with 5 entries:
34+
"driver" => "n5"
35+
"create" => true
36+
"kvstore" => Dict("driver"=>"file", "path"=>"tmp/dataset/")
37+
"metadata" => Dict{String, Any}("blockSize"=>[100, 100], "dataType"=>"…
38+
"delete_existing" => true
39+
40+
julia> A = PyTensorStore.open(d).result()
41+
PyTensorStore.TensorStoreWrapper(<py TensorStore({
42+
'context': {
43+
'cache_pool': {},
44+
'data_copy_concurrency': {},
45+
'file_io_concurrency': {},
46+
'file_io_sync': True,
47+
},
48+
'driver': 'n5',
49+
'dtype': 'uint32',
50+
'kvstore': {'driver': 'file', 'path': 'tmp/dataset/'},
51+
'metadata': {
52+
'blockSize': [100, 100],
53+
'compression': {'level': -1, 'type': 'gzip', 'useZlib': False},
54+
'dataType': 'uint32',
55+
'dimensions': [1000, 20000],
56+
},
57+
'transform': {
58+
'input_exclusive_max': [[1000], [20000]],
59+
'input_inclusive_min': [0, 0],
60+
},
61+
})>)
62+
63+
julia> A[1:100, 1:100]
64+
PyTensorStore.TensorStoreWrapper(<py TensorStore({
65+
'context': {
66+
'cache_pool': {},
67+
'data_copy_concurrency': {},
68+
'file_io_concurrency': {},
69+
'file_io_sync': True,
70+
},
71+
'driver': 'n5',
72+
'dtype': 'uint32',
73+
'kvstore': {'driver': 'file', 'path': 'tmp/dataset/'},
74+
'metadata': {
75+
'blockSize': [100, 100],
76+
'compression': {'level': -1, 'type': 'gzip', 'useZlib': False},
77+
'dataType': 'uint32',
78+
'dimensions': [1000, 20000],
79+
},
80+
'transform': {
81+
'input_exclusive_max': [100, 100],
82+
'input_inclusive_min': [0, 0],
83+
},
84+
})>)
85+
86+
julia> A[1:100, 1:100].write(ones(UInt32, 100, 100)*UInt32(5)).result()
87+
Python: None
88+
89+
julia> A[1:100, 1:100].read().result()
90+
100×100 PyArray{UInt32, 2}:
91+
0x00000005 0x00000005 0x00000005 … 0x00000005 0x00000005 0x00000005
92+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
93+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
94+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
95+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
96+
0x00000005 0x00000005 0x00000005 … 0x00000005 0x00000005 0x00000005
97+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
98+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
99+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
100+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
101+
0x00000005 0x00000005 0x00000005 … 0x00000005 0x00000005 0x00000005
102+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
103+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
104+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
105+
⋮ ⋱
106+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
107+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
108+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
109+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
110+
0x00000005 0x00000005 0x00000005 … 0x00000005 0x00000005 0x00000005
111+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
112+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
113+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
114+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
115+
0x00000005 0x00000005 0x00000005 … 0x00000005 0x00000005 0x00000005
116+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
117+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
118+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
119+
0x00000005 0x00000005 0x00000005 0x00000005 0x00000005 0x00000005
120+
121+
julia> A[1,1].write(9).result()
122+
Python: None
123+
124+
julia> A[1,1].read().result()
125+
0-dimensional PyArray{UInt32, 0}:
126+
0x00000009
127+
```

‎src/FutureWrapper.jl‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
struct FutureWrapper{Result}
2+
parent::Py
3+
end
4+
Base.parent(w::FutureWrapper) = getfield(w, :parent)
5+
PythonCall.Py(w::FutureWrapper) = parent(w)
6+
function Base.getproperty(w::FutureWrapper, s::Symbol)
7+
s == :result ? (x...,) -> result(w, x...) :
8+
getproperty(parent(w), s)
9+
end
10+
result(w::FutureWrapper{Result}, x...) where Result = Result(parent(w).result(x...))
11+
pyarray_nocopy(x) = PyArray(x; copy=false)

‎src/PyTensorStore.jl‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
module PyTensorStore
22

3-
# Write your package code here.
3+
using PythonCall
4+
5+
public open
6+
7+
include("Python/Python.jl")
8+
include("TensorStoreWrapper.jl")
9+
include("FutureWrapper.jl")
10+
11+
open(spec; kwargs...) = FutureWrapper{TensorStoreWrapper}(Python.open(spec; kwargs...))
412

513
end

‎src/Python/Python.jl‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
module Python
2+
using PythonCall
3+
4+
pyts() = @pyconst(pyimport("tensorstore"))
5+
6+
open(spec::Py; kwargs...) = pyts().open(spec; kwargs...)
7+
open(spec::AbstractDict) = pyts().open(pydict_recursive(spec))
8+
pydict_recursive(x) = x
9+
pydict_recursive(x::AbstractArray) = pylist(x)
10+
function pydict_recursive(x::AbstractDict)
11+
return pydict(Dict(
12+
string.(keys(x)) .=> pydict_recursive.(values(x))
13+
))
14+
end
15+
end

‎src/TensorStoreWrapper.jl‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
struct TensorStoreWrapper
2+
parent::Py
3+
end
4+
Base.parent(w::TensorStoreWrapper) = getfield(w, :parent)
5+
PythonCall.Py(w::TensorStoreWrapper) = parent(w)
6+
function Base.propertynames(w::TensorStoreWrapper)
7+
propertynames(parent(w))
8+
end
9+
function Base.getproperty(w::TensorStoreWrapper, s::Symbol)
10+
result = s == :read ? (x...,) -> FutureWrapper{pyarray_nocopy}(parent(w).read(x...)) :
11+
getproperty(parent(w), s)
12+
return pyconvert(Any, result)
13+
end
14+
function Base.getindex(w::TensorStoreWrapper, indices...)
15+
@debug "getindex" indices
16+
py_indices = tsindex.(indices)
17+
@debug "py_indices" py_indices
18+
TensorStoreWrapper(parent(w)[py_indices...])
19+
end
20+
Base.size(w::TensorStoreWrapper) = pyconvert(Tuple, w.shape)
21+
22+
# Convert indicies into TensorStoreWrapper into Python indexes
23+
tsindex(x) = x
24+
tsindex(::Colon) = pyslice(nothing)
25+
tsindex(i::Integer) = i-1
26+
tsindex(r::UnitRange) = pyslice(first(r)-1, last(r))
27+
tsindex(r::AbstractRange) = pyslice(first(r)-1, last(r), step(r))
28+
tsindex(ci::CartesianIndex) = Tuple(ci - CartesianIndex(1,1))
29+

0 commit comments

Comments
 (0)