We will the two basic examples for a dense and sparse array simply to create array data on disk to refer to later in examples that follow.
library(tiledb)
tdir <- tempdir()
uridense <- file.path(tdir, "dense")
uridensefix <- file.path(tdir, "densefix")
uridensevar <- file.path(tdir, "densevar")
create_array <- function(array_name) {
# Check if the array already exists.
if (tiledb_object_type(array_name) == "ARRAY") {
message("Array already exists.")
return(invisible(NULL))
}
# The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4].
dom <- tiledb_domain(dims = c(tiledb_dim("rows", c(1L, 4L), 4L, "INT32"),
tiledb_dim("cols", c(1L, 4L), 4L, "INT32")))
# The array will be dense with a single attribute "a" so each (i,j) cell can store an integer.
schema <- tiledb_array_schema(dom, attrs = tiledb_attr("a", type = "INT32"))
# Create the (empty) array on disk.
tiledb_array_create(array_name, schema)
}
write_array <- function(array_name) {
data <- array(c(c(1L, 5L, 9L, 13L),
c(2L, 6L, 10L, 14L),
c(3L, 7L, 11L, 15L),
c(4L, 8L, 12L, 16L)), dim = c(4,4))
# Open the array and write to it.
A <- tiledb_dense(uri = array_name)
A[] <- data
}
create_array(uridense)
## [1] "/tmp/RtmpIIqHdF/dense"
write_array(uridense)
urisparse <- file.path(tdir, "sparse")
create_array <- function(array_name) {
# Check if the array already exists.
if (tiledb_object_type(array_name) == "ARRAY") {
message("Array already exists.")
return(invisible(NULL))
}
# The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4].
dom <- tiledb_domain(dims = c(tiledb_dim("rows", c(1L, 4L), 4L, "INT32"),
tiledb_dim("cols", c(1L, 4L), 4L, "INT32")))
# The array will be dense with a single attribute "a" so each (i,j) cell can store an integer.
schema = tiledb_array_schema(dom, attrs=tiledb_attr("a", type = "INT32"), sparse = TRUE)
# Create the (empty) array on disk.
tiledb_array_create(array_name, schema)
}
write_array <- function(array_name) {
I <- c(1, 2, 2)
J <- c(1, 4, 3)
data <- c(1L, 2L, 3L)
# Open the array and write to it.
A <- tiledb_sparse(uri = array_name)
A[I, J] <- data
}
create_array(urisparse)
## [1] "/tmp/RtmpIIqHdF/sparse"
write_array(urisparse)
close_and_reopen <- function(arr, txt) {
res <- tiledb:::libtiledb_array_close(arr@ptr)
res <- tiledb:::libtiledb_array_open_with_ptr(arr@ptr, txt)
}
library(tiledb)
# Create dimension
# In C++: Dimension::create<T>(...)
dim <- tiledb_dim("dim", c(1L, 4L), 2L, "INT32")
library(tiledb)
# .. create dimensions `dim1`, `dim2`
dim1 <- tiledb_dim("dim1", c(1L, 4L), 2L, "INT32")
dim2 <- tiledb_dim("dim2", c(1L, 2L), 2L, "INT32")
# Create domain with two dimensions
# In C++: domain.add_dimensions(dim1).add_dimension(dim2)
dom <- tiledb_domain(dims = c(dim1, dim2))
# Create attribute
# In C++: Attribute::create<T>(ctx, "attr")
attr <- tiledb_attr("attr", "INT32")
# We can also set the number of attributes to a value other than one
tiledb:::libtiledb_attribute_set_cell_val_num(attr@ptr, 3)
# And we can set the a variable number of attributes (using NA to select variable size)
tiledb:::libtiledb_attribute_set_cell_val_num(attr@ptr, NA)
## TODO: R function for level setter
comp <- tiledb_filter("GZIP")
tiledb_filter_set_option(comp,"COMPRESSION_LEVEL", 10)
# Create a filter list with the compressor
filter_list <- tiledb_filter_list(comp)
# Create attribute with the filter list
attr <- tiledb_attr("attr", "INT32", filter_list = filter_list)
# Create filters
f1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
f2 <- tiledb_filter("ZSTD")
# Create a filter list with the two filters
filter_list <- tiledb_filter_list(c(f1,f2))
# Create attribute with the filter list
attr <- tiledb_attr("attr", "INT32", filter_list = filter_list)
# ... create filter list
set_max_chunk_size(filter_list, 10000)
# ... create domain dom
attr1 <- tiledb_attr("attr1", "INT32", filter_list = filter_list)
attr2 <- tiledb_attr("attr2", "FLOAT64", filter_list = filter_list)
# Create a dense array
schema <- tiledb_array_schema(dom, c(attr1, attr2), sparse = FALSE)
# Or, create a spaese array
# schema <- tiledb_array_schema(dom, c(attr1, attr2), sparse = TRUE)
# ... create domain dom
# ... create attributes attr1, attr2
# The tile and order can be "ROW_MAJOR" or "COL_MAJOR"
schema <- tiledb_array_schema(dom, c(attr1, attr2),
cell_order = "COL_MAJOR",
tile_order = "ROW_MAJOR")
tiledb:::libtiledb_array_schema_set_capacity(schema@ptr, 100000)
tiledb:::libtiledb_array_schema_get_capacity(schema@ptr)
## [1] 100000
## TODO higher-level R functions
# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter lists fl1, fl2, similar to attributes
f1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
f2 <- tiledb_filter("ZSTD")
fl1 <- tiledb_filter_list(c(f1))
fl2 <- tiledb_filter_list(c(f2))
# Create the schema setting the coordinates and offsets filter lists
schema <- tiledb_array_schema(dom, c(attr1, attr2),
coords_filter_list = fl1,
offsets_filter_list = fl2)
tiledb:::libtiledb_array_schema_check(schema@ptr)
## TODO higher-level R functions
# ... create array schema
# Create the array
tiledb_array_create(uridense, schema)
encryption_key <- "0123456789abcdeF0123456789abcdeF"
uridensewkey <- file.path(tdir, "densewkey")
if (dir.exists(uridensewkey)) unlink(uridensewkey, recursive=TRUE)
# shortcut: borrow schema from dense array; could create schema too
schema <- schema(tiledb_dense(uridense))
tiledb:::libtiledb_array_create_with_key(uridensewkey, schema@ptr, encryption_key)
## [1] "/tmp/RtmpIIqHdF/densewkey"
##TODO: R support
## prepare a larger 5 x 5 to embed into
tmp <- tempfile()
d1 <- tiledb_dim("d1", domain = c(1L, 5L))
d2 <- tiledb_dim("d2", domain = c(1L, 5L))
dom <- tiledb_domain(c(d1, d2))
val <- tiledb_attr(name="val", type = "INT32")
sch <- tiledb_array_schema(dom, c(val))
tiledb_array_create(tmp, sch)
dat <- matrix(as.integer(rnorm(25)*100), 5, 5)
arr <- tiledb_dense(tmp, as.data.frame=FALSE)
arr[] <- dat
# Prepare a 2x3 dense array
# Contrary to Python, R by default stores arrays in col-major order
data <- array(c(1L, 4L, 2L, 5L, 3L, 6L), dim=c(2,3))
# Prepare the [1,2] x [2,4] subarray to write to
I <- c(1:2)
J <- c(2:4)
# Open the array and write the data to it
A <- tiledb_dense(uri = tmp)
A[I, J] <- data
unlink(tmp, recursive=TRUE)
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridense, "WRITE")
## data: simple (integer sequence) of 1:16 times 10
vec <- 1:16 * 10L
subarr <- c(1L,4L, 1L,4L)
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "WRITE")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", vec)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
res <- tiledb:::libtiledb_array_close(arrptr)
tmp <- urisparse
unlink(tmp, recursive=TRUE)
d1 <- tiledb_dim("d1", domain = c(1L, 5L))
d2 <- tiledb_dim("d2", domain = c(1L, 5L))
dom <- tiledb_domain(c(d1, d2))
val <- tiledb_attr("val", type = "INT32")
sch <- tiledb_array_schema(dom, val, sparse=TRUE)
tiledb_array_create(tmp, sch)
## [1] "/tmp/RtmpIIqHdF/sparse"
# Prepare some data
data <- c(3L, 4L, 1L, 2L)
I <- c(3, 4, 1, 2)
J <- c(3, 4, 2, 1)
# Open the array and write the data to it
A <- tiledb_sparse(uri = tmp)
A[I, J] <- data
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open_with_key(ctx@ptr, uridensewkey, "WRITE", encryption_key)
## data: simple (integer sequence) of 1:16 times 10
vec <- 1:16 * 10L
subarr <- c(1L,4L, 1L,4L)
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "WRITE")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", vec)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
res <- tiledb:::libtiledb_array_close(arrptr)
# TODO Higher-level R support
if (dir.exists(uridensefix)) unlink(uridensefix, recursive=TRUE)
d1 <- tiledb_dim("d1", domain = c(1L, 4L))
d2 <- tiledb_dim("d2", domain = c(1L, 4L))
dom <- tiledb_domain(c(d1, d2))
vec <- 1:32 * 10L
attr <- tiledb_attr("a", type = r_to_tiledb_type(vec))
## set to two values per cell
tiledb:::libtiledb_attribute_set_cell_val_num(attr@ptr, 2)
sch <- tiledb_array_schema(dom, attr)
tiledb_array_create(uridensefix, sch)
## [1] "/tmp/RtmpIIqHdF/densefix"
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridensefix, "WRITE")
subarr <- c(1L,4L, 1L,4L)
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "WRITE")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", vec)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
res <- tiledb:::libtiledb_array_close(arrptr)
#TODO Higher-level R support
if (dir.exists(uridensevar)) unlink(uridensevar, recursive=TRUE)
## Define array
## The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4].
dom <- tiledb_domain(dims = c(tiledb_dim("rows", c(1L, 4L), 4L, "INT32"),
tiledb_dim("cols", c(1L, 4L), 4L, "INT32")))
attr <- tiledb_attr("a1", type = "CHAR")
## set to variable length
tiledb:::libtiledb_attribute_set_cell_val_num(attr@ptr, NA)
## now set the schema
ctx <- tiledb_ctx()
schptr <- tiledb:::libtiledb_array_schema_create(ctx@ptr, "DENSE")
tiledb:::libtiledb_array_schema_set_domain(schptr, dom@ptr)
tiledb:::libtiledb_array_schema_set_cell_order(schptr, "ROW_MAJOR")
tiledb:::libtiledb_array_schema_set_tile_order(schptr, "ROW_MAJOR")
tiledb:::libtiledb_array_schema_add_attribute(schptr, attr@ptr)
## Create the (empty) array on disk.
tiledb:::libtiledb_array_create(uridensevar, schptr)
## [1] "/tmp/RtmpIIqHdF/densevar"
data <- "abbcccddeeefghhhijjjkklmnoop";
offsets <- c(0L, 1L, 3L, 6L, 8L, 11L, 12L, 13L, 16L, 17L, 20L, 22L, 23L, 24L, 25L, 27L)
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridensevar, "WRITE")
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "WRITE")
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
bufptr <- tiledb:::libtiledb_query_buffer_var_char_create(offsets, data)
qryptr <- tiledb:::libtiledb_query_set_buffer_var_char(qryptr, "a1", bufptr)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
tiledb:::libtiledb_array_close(arrptr)
## <pointer: 0x563edec119a0>
#TODO Higher-level R support
## continuing from previous example on dense variable length array
## (but this works of course with any array after a write is needed
numfrag <- tiledb:::libtiledb_query_get_fragment_num(qryptr)
uritxt <- tiledb:::libtiledb_query_get_fragment_uri(qryptr, 0)
##TODO Higher-level R support
array_name <- urisparse
A <- tiledb_sparse(uri = array_name)
# can also load dense
sch <- schema(A)
# can also open encrypted arrays with key
array_name <- urisparse
# Get array schema
A <- tiledb_sparse(uri = array_name)
# or use tiledb_dense(uri = array_name) for dense
schema <- schema(A)
# Get array type
sparse <- is.sparse(schema)
# Get tile capacity
tiledb:::libtiledb_array_schema_get_capacity(sch@ptr)
## [1] 10000
# TODO: R accessor for tile
# Get tile order
t_order <- tile_order(schema)
# Get cell order
c_order <- cell_order(schema)
# Get coordinates and offset filter list
reslist <- filter_list(schema)
# Get the array domain
dom <- domain(schema)
# Get all attributes as list
attrs <- attrs(schema)
# Get attribute from name
attr <- attrs(schema, "attr")
# Dump the array schema in ASCII format to stdout
show(schema)
# ... get array schema
# ... get domain from schema
dom <- domain(schema)
# Get the domain datatype (i.e., the datatype of all dimensions)
type <- datatype(dom)
# Get number of dimensions
dim_num <- dim(dom)
# Get all dimension
dims <- dimensions(dom)
# Dump the domain in ASCII format in the selected output
show(dom)
# ... get array schema
# ... get domain
# ... get dimension by index or name
#FIXME
# Get dimension name
dim_name <- name(dim)
# Get dimension datatype
dim_type <- datatype(dim)
# Get dimension domain
domain <- domain(dim)
# Get tile extent
tile_extent <- tile(dim)
# Dump the dimension in ASCII format in the selected output
show(dim)
# ... get array schema
# ... get attribute by index or name
stopifnot(is.null(attr))
# Get attribute name
attr_name <- name(attr)
# Get attribute datatype
attr_type <- datatype(attr)
# Get filter list
filter_list <- filter_list(attr)
# Check if attribute is variable-length
# variable length attribtues are not supported in R currently
# Get number of values per cell
num <- ncells(attr)
# Get cell size for this attribute
sz <-- tiledb:::libtiledb_attribute_get_cell_val(attr@ptr)
# Dump the attribute in ASCII format in the selected output
show(attr)
fl <- tiledb_filter_list()
# get number of filter
tiledb:::libtiledb_filter_list_get_nfilters(fl@ptr)
# get max chunk size
tiledb:::libtiledb_filter_list_get_max_chunk_size(fl@ptr)
# get filter from index
ind <- i
tiledb:::libtiledb_filter_list_get_filter_from_index(fl@ptr, ind)
# Create a TileDB context
ctx <- tiledb_ctx()
# Open a dense array
A <- tiledb_dense(uri = uridense, ctx=ctx)
# Or, open a sparse array
# A <- tiledb_sparse(uri = "<array-uri>", ctx=ctx)
# Slice only rows 1, 2 and cols 2, 3, 4
data <- A[1:2, 2:4]
show(data)
## [,1] [,2] [,3]
## [1,] 20 30 40
## [2,] 60 70 80
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridense, "READ")
## subarray of rows 1,2 and cols 2,3,4
subarr <- c(1L,2L, 2L,4L)
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
v <- integer(6) # reserve space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", v)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
print(v) # unformed array, no coordinates
## [1] 20 30 40 60 70 80
res <- tiledb:::libtiledb_array_close(arrptr)
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridensevar, "READ")
subarr <- c(1L,4L, 1L,4L)
bufptr <- tiledb:::libtiledb_query_buffer_var_char_alloc(arrptr, subarr, "a1", 16, 100)
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
qryptr <- tiledb:::libtiledb_query_set_buffer_var_char(qryptr, "a1", bufptr)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
tiledb:::libtiledb_array_close(arrptr)
## <pointer: 0x563edf9bb160>
mat <- tiledb:::libtiledb_query_get_buffer_var_char(bufptr)
print(mat, quote=FALSE)
## [,1] [,2] [,3] [,4]
## [1,] a eee i m
## [2,] bb f jjj n
## [3,] ccc g kk oo
## [4,] dd hhh l p
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridensevar, "READ")
# returns a list object
reslist <- tiledb:::libtiledb_array_nonempty_domain(arrptr)
tiledb:::libtiledb_array_close(arrptr)
## <pointer: 0x563edcaf1e60>
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open_with_key(ctx@ptr, uridensewkey, "READ", encryption_key)
## subarray of rows 1,2 and cols 2,3,4
subarr <- c(1L,2L, 2L,4L)
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
v <- integer(6) # reserve space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", v)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
print(v) # unformed array, no coordinates
## [1] 20 30 40 60 70 80
res <- tiledb:::libtiledb_array_close(arrptr)
## TODO open at timestamp example
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridense, "READ")
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
## range of rows 1 and 2, and 4 for dim 1, all rows for dim 2
qryptr <- tiledb:::libtiledb_query_add_range(qryptr, 0, 1L, 2L)
qryptr <- tiledb:::libtiledb_query_add_range(qryptr, 0, 4L, 4L)
qryptr <- tiledb:::libtiledb_query_add_range(qryptr, 1, 1L, 4L)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
v <- integer(12) # reserve space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", v)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
print(v) # unformed array, no coordinates
## [1] 10 20 30 40 50 60 70 80 130 140 150 160
res <- tiledb:::libtiledb_array_close(arrptr)
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridense, "READ")
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
subarr <- c(1L,4L, 1L,4L)
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
vec <- integer(4) # reserve (insufficient) space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", vec)
finished <- FALSE
while (!finished) {
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
print(vec)
finished <- tiledb:::libtiledb_query_status(qryptr) == "COMPLETE"
}
## [1] 10 20 30 40
## [1] 50 60 70 80
## [1] 90 100 110 120
## [1] 130 140 150 160
res <- tiledb:::libtiledb_array_close(arrptr)
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridensevar, "READ")
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
estsz <- tiledb:::libtiledb_query_get_est_result_size_var(qryptr, "a1")
res <- tiledb:::libtiledb_array_close(arrptr)
estsz
## [1] 16 28
ctx <- tiledb_ctx()
tstamp <- Sys.time() - 1.0; # one seconds ago (as the arrays created here are in temp.files)
arrptr <- tiledb:::libtiledb_array_open_at(ctx@ptr, uridense, "READ", tstamp)
subarr <- c(1L,2L, 2L,4L)
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
v <- integer(6) # reserve space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", v)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
res <- tiledb:::libtiledb_array_close(arrptr)
v
## [1] 0 0 0 0 0 0
arr <- tiledb_dense(uridense)
close_and_reopen(arr, "WRITE")
res <- tiledb_put_metadata(arr, "aaa", 100L)
res <- tiledb_put_metadata(arr, "bb", c(1.1, 2.2))
invisible(tiledb:::libtiledb_array_close(arr@ptr))
One can read by key:
close_and_reopen(arr, "READ")
tiledb_get_metadata(arr, "aaa")
## [1] 100
Or one can retrieve all metadata at once:
md <- tiledb_get_all_metadata(arr)
print(md)
## aaa: 100
## bb: 1.1, 2.2
close_and_reopen(arr, "WRITE")
res <- tiledb_delete_metadata(arr, "aaa")