import datetime

import typing

import pathlib

import storage

from . import schemas

from . import types

T = typing.TypeVar('T')

DatasetType = typing.TypeVar('DatasetType', bound='Dataset')

__all__ = [

"__version__",

"AgreementError",

"AgreementNotAcceptedError",

"Array",

"AuthenticationError",

"AuthorizationError",

"BadRequestError",

"Branch",

"BranchExistsError",

"BranchNotFoundError",

"BranchView",

"Branches",

"BranchesView",

"BytePositionIndexOutOfChunk",

"CanNotCreateTensorWithProvidedCompressions",

"CannotDeleteMainBranchError",

"CannotRenameMainBranchError",

"Client",

"Column",

"ColumnAlreadyExistsError",

"ColumnDefinition",

"ColumnDefinitionView",

"ColumnDoesNotExistError",

"ColumnMissingAppendValueError",

"ColumnStatistics",

"ColumnView",

"CredsKeyAlreadyAssignedError",

"Dataset",

"DatasetUnavailableError",

"DatasetView",

"DimensionsMismatch",

"DimensionsMismatchError",

"DtypeMismatch",

"EmbeddingSizeMismatch",

"EmptyColumnNameError",

"Executor",

"ExplainQueryResult",

"ExpiredTokenError",

"FormatNotSupportedError",

"Future",

"FutureVoid",

"GcsStorageProviderFailed",

"HTTPBodyIsMissingError",

"HTTPBodyIsNotJSONError",

"HTTPRequestFailedError",

"History",

"IncorrectDeeplakePathError",

"IndexAlreadyExistsError",

"IndexBuildConfig",

"IndexingMode",

"InvalidBinaryMaskCompression",

"InvalidChunkStrategyType",

"InvalidColumnValueError",

"InvalidCredsKeyAssignmentError",

"InvalidImageCompression",

"InvalidTextCompression",

"InvalidIndexCreationError",

"InvalidLinkDataError",

"InvalidLinkType",

"InvalidMedicalCompression",

"InvalidPolygonShapeError",

"InvalidSegmentMaskCompression",

"InvalidSequenceOfSequence",

"InvalidTextType",

"InvalidType",

"InvalidTypeAndFormatPair",

"InvalidTypeDimensions",

"InvalidURIError",

"JSONIndexNotFound",

"JSONKeyNotFound",

"LogExistsError",

"LogNotexistsError",

"Metadata",

"NotFoundError",

"NotLoggedInAgreementError",

"PermissionDeniedError",

"PushError",

"QuantizationType",

"Random",

"random",

"ReadOnlyDataset",

"ReadOnlyDatasetModificationError",

"ReadOnlyMetadata",

"Row",

"RowRange",

"RowRangeView",

"RowView",

"Schema",

"SchemaView",

"SearchConfig",

"ShapeIndexOutOfChunk",

"StorageAccessDenied",

"StorageInternalError",

"StorageKeyAlreadyExists",

"StorageKeyNotFound",

"StorageNetworkConnectionError",

"StorageProviderMissingError",

"Tag",

"TagExistsError",

"TagNotFoundError",

"TagView",

"Tags",

"TagsView",

"TensorAlreadyExists",

"UnevenColumnsError",

"UnevenUpdateError",

"UnexpectedInputDataForDicomColumn",

"UnexpectedMedicalTypeInputData",

"UnknownBoundingBoxCoordinateFormat",

"UnknownBoundingBoxPixelFormat",

"UnknownFormat",

"UnknownStringType",

"UnknownType",

"UnspecifiedDtype",

"UnsupportedChunkCompression",

"UnsupportedPythonType",

"UnsupportedSampleCompression",

"Version",

"VersionNotFoundError",

"WriteFailedError",

"WrongChunkCompression",

"WrongSampleCompression",

"__prepare_atfork",

"client",

"connect",

"convert",

"copy",

"core",

"create",

"create_async",

"_create_global_cache",

"delete",

"delete_async",

"disconnect",

"exists",

"exists_async",

"explain_query",

"from_coco",

"from_csv",

"from_parquet",

"like",

"link",

"link_async",

"open",

"open_async",

"open_read_only",

"open_read_only_async",

"prepare_query",

"query",

"query_async",

"replay_log",

"schemas",

"storage",

"tql",

"types",

"TelemetryClient",

"telemetry_client"

]

class Future(typing.Generic[T]):

"""

A future representing an asynchronous operation result in ML pipelines.

The Future class enables non-blocking operations for data loading and processing,

particularly useful when working with large ML datasets or distributed training.

Once resolved, the Future holds the operation result which can be accessed either

synchronously or asynchronously.

Methods:

result() -> typing.Any:

Blocks until the Future resolves and returns the result.

__await__() -> typing.Any:

Enables using the Future in async/await syntax.

cancel() -> None:

Cancels the Future if it is still pending.

is_completed() -> bool:

Checks if the Future has resolved without blocking.

<!-- test-context

```python

import deeplake

ds = deeplake.create("mem://ml-data/embeddings")

ds = deeplake.create("mem://ml-data/images")

ds.add_column("images", "int32")

ds.append({"images": [0] * 300})

deeplake.open_async = lambda x: deeplake._deeplake.open_async(x.replace("s3://", "mem://"))

```

-->

Examples:

Loading ML dataset asynchronously:

```python

future = deeplake.open_async("s3://ml-data/embeddings")

# Check status without blocking

if not future.is_completed():

print("Still loading...")

# Block until ready

ds = future.result()

```

Using with async/await:

```python

async def load_data():

ds = await deeplake.open_async("s3://ml-data/images")

batch = await ds["images"].get_async(slice(0, 32))

return batch

```

"""

def result(self) -> T:

"""

Blocks until the Future resolves and returns the result.

Returns:

typing.Any: The operation result once resolved.

<!-- test-context

```python

import deeplake

import numpy as np

ds = deeplake.create("tmp://")

ds.add_column("images", "int32")

ds.append({"images": [0] * 300})

```

-->

Examples:

```python

future = ds["images"].get_async(slice(0, 32))

batch = future.result() # Blocks until batch is loaded

```

"""

...

def __await__(self) -> typing.Any:

"""

Makes the Future compatible with async/await syntax.

<!-- test-context

```python

import deeplake

import numpy as np

ds = deeplake.create("tmp://")

ds.add_column("images", "int32")

ds.append({"images": [0] * 300})

```

-->

Examples:

```python

async def load_batch():

batch = await ds["images"].get_async(slice(0, 32))

```

Returns:

typing.Any: The operation result once resolved.

"""

...

def is_completed(self) -> bool:

"""

Checks if the Future has resolved without blocking.

Returns:

bool: True if resolved, False if still pending.

<!-- test-context

```python

import deeplake

import numpy as np

ds = deeplake.create("tmp://")

ds.add_column("label", "int32")

ds.append({"label": [0] * 300})

ds["label"].metadata["class_names"] = ["car", "truck", "bus"]

```

-->

Examples:

```python

future = ds.query_async("SELECT * WHERE label = 'car'")

if future.is_completed():

results = future.result()

else:

print("Query still running...")

```

"""

...

def cancel(self) -> None:

"""

Cancels the Future if it is still pending.

"""

class FutureVoid:

"""

A Future representing a void async operation in ML pipelines.

Similar to Future but for operations that don't return values, like saving

or committing changes. Useful for non-blocking data management operations.

<!-- test-context

```python

import deeplake

ds = deeplake.create("tmp://")

ds.add_column("embeddings", "float32")

ds.append({"embeddings": [0.1] * 100})

new_embeddings = [0.2] * 32

def process_other_data():

pass

```

-->

Examples:

Asynchronous dataset updates:

```python

# Update embeddings without blocking

future = ds["embeddings"].set_async(slice(0, 32), new_embeddings)

# Do other work while update happens

process_other_data()

# Wait for update to complete

future.wait()

```

Using with async/await:

```python

async def update_dataset():

await ds.commit_async()

print("Changes saved")

```

"""

def wait(self) -> None:

"""

Blocks until the operation completes.

<!-- test-context

```python

import deeplake

ds = deeplake.create("tmp://")

```

-->

Examples:

```python

future = ds.commit_async()

future.wait() # Blocks until commit finishes

```

"""

...

def __await__(self) -> typing.Any:

"""

Makes the FutureVoid compatible with async/await syntax.

<!-- test-context

```python

import deeplake

ds = deeplake.create("tmp://")

```

-->

Examples:

```python

async def save_changes():

await ds.commit_async()

```

"""

...

def cancel(self) -> None:

"""

Cancels the Future if it is still pending.

"""

def is_completed(self) -> bool:

"""

Checks if the operation has completed without blocking.

Returns:

bool: True if completed, False if still running.

<!-- test-context

```python

import deeplake

ds = deeplake.create("tmp://")

```

-->

Examples:

```python

future = ds.commit_async()

if future.is_completed():

print("Commit finished")

else:

print("Commit still running...")

```

"""

...

class Array:

"""

Wrapper around n dimensional array

"""

@property

def dtype(self) -> numpy.dtype[typing.Any]:

"""

Returns the data type of the array

"""

...

@property

def shape(self) -> tuple:

"""

Returns the shape of the array

"""

...

def __getitem__(self, index: int | slice | list | tuple ) -> typing.Any:

"""

Returns the value at the given index or slice

"""

...

def get_async(self, index: int | slice | list | tuple) -> Future[typing.Any]:

"""

Returns the value at the given index or slice asynchronously

"""

...

def __str__(self) -> str: ...

def __repr__(self) -> str: ...

class ReadOnlyMetadata:

"""

Read-only access to dataset and column metadata for ML workflows.

Stores important information about datasets like:

- Model parameters and hyperparameters

- Preprocessing statistics (mean, std, etc.)

- Data splits and fold definitions

- Version and training information

<!-- test-context

```python

import deeplake

ds = deeplake.create("tmp://")

ds.add_column("images", "int32")

ds.metadata["model_name"] = "resnet50"

ds.metadata["hyperparameters"] = {"learning_rate": 0.001, "batch_size": 32}

ds["images"].metadata["mean"] = [0.485, 0.456, 0.406]

ds["images"].metadata["std"] = [0.229, 0.224, 0.225]

```

-->

Examples:

Accessing model metadata:

```python

metadata = ds.metadata

model_name = metadata["model_name"]

model_params = metadata["hyperparameters"]

```

Reading preprocessing stats:

```python

mean = ds["images"].metadata["mean"]

std = ds["images"].metadata["std"]

```

"""

def __getitem__(self, key: str) -> typing.Any:

"""

Gets metadata value for the given key.

Args:

key: Metadata key to retrieve

Returns:

The stored metadata value

<!-- test-context

```python

import deeplake

ds = deeplake.create("tmp://")

ds.add_column("images", "int32")

ds.metadata["model_name"] = "resnet50"

ds.metadata["hyperparameters"] = {"learning_rate": 0.001, "batch_size": 32}

ds["images"].metadata["mean"] = [0.485, 0.456, 0.406]

ds["images"].metadata["std"] = [0.229, 0.224, 0.225]

```

-->

Examples:

```python

mean = ds["images"].metadata["mean"]

std = ds["images"].metadata["std"]

```

"""

...

def keys(self) -> list[str]:

"""

Lists all available metadata keys.

Returns:

list[str]: List of metadata key names

<!-- test-context

```python

import deeplake

ds = deeplake.create("tmp://")

ds.add_column("images", "int32")

metadata = ds.metadata

```

-->

Examples:

```python

# Print all metadata

for key in metadata.keys():

print(f"{key}: {metadata[key]}")

```

"""

...

def __contains__(self, key: str) -> bool:

"""

Checks if the metadata contains the given key.

"""

...

class Metadata(ReadOnlyMetadata):

"""

Writable access to dataset and column metadata for ML workflows.

Stores important information about datasets like:

- Model parameters and hyperparameters

- Preprocessing statistics

- Data splits and fold definitions

- Version and training information

Changes are persisted immediately without requiring `commit()`.

<!-- test-context

```python

import deeplake

ds = deeplake.create("tmp://")

ds.add_column("images", "int32")

```

-->

Examples:

Storing model metadata:

```python

ds.metadata["model_name"] = "resnet50"

ds.metadata["hyperparameters"] = {

"learning_rate": 0.001,

"batch_size": 32

}

```

Setting preprocessing stats:

```python

ds["images"].metadata["mean"] = [0.485, 0.456, 0.406]

ds["images"].metadata["std"] = [0.229, 0.224, 0.225]

```

"""

def __setitem__(self, key: str, value: typing.Any) -> None:

"""

Sets metadata value for given key. Changes are persisted immediately.

Args:

key: Metadata key to set

value: Value to store

Examples:

```python

ds.metadata["train_split"] = 0.8

ds.metadata["val_split"] = 0.1

ds.metadata["test_split"] = 0.1

```

"""

...

class ExplainQueryResult:

def __str__(self) -> str:

...

def to_dict(self) -> typing.Any:

...

def prepare_query(query: str, token: str | None = None, creds: dict[str, str] | None = None) -> Executor:

"""

Prepares a TQL query for execution with optional authentication.

Args:

query: TQL query string to execute

token: Optional Activeloop authentication token

creds (dict, optional): Dictionary containing credentials used to access the dataset at the path.

Returns:

Executor: An executor object to run the query.

<!-- test-context

```python

import deeplake

ds = deeplake.create("mem://parametriized")

ds.add_column("category", "text")

ds.append({"category": ["active", "inactive", "not sure"]})

ds.commit()

```

-->

Examples:

Running a parametrized batch query:

```python

ex = deeplake.prepare_query('SELECT * FROM "mem://parametriized" WHERE category = ?')

results = ex.run_batch([["active"], ["inactive"]])

assert len(results) == 2

```

"""

...

def query(query: str, token: str | None = None, creds: dict[str, str] | None = None) -> DatasetView:

"""

Executes TQL queries optimized for ML data filtering and search.

TQL is a SQL-like query language designed for ML datasets, supporting:

- Vector similarity search

- Text semantic search

- Complex data filtering

- Joining across datasets

- Efficient sorting and pagination

Args:

query: TQL query string supporting:

- Vector similarity: COSINE_SIMILARITY, L2_NORM

- Text search: BM25_SIMILARITY, CONTAINS

- MAXSIM similarity for ColPali embeddings: MAXSIM

- Filtering: WHERE clauses

- Sorting: ORDER BY

- Joins: JOIN across datasets

token: Optional Activeloop authentication token

creds (dict, optional): Dictionary containing credentials used to access the dataset at the path.

- If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.

- It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.

- If nothing is given is, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets

Returns:

DatasetView: Query results that can be:

- Used directly in ML training

- Further filtered with additional queries

- Converted to PyTorch/TensorFlow dataloaders

- Materialized into a new dataset

<!-- test-context

```python

import deeplake

ds = deeplake.create("mem://embeddings")

ds.add_column("vector", deeplake.types.Array("float32", 1))

ds.commit()

ds = deeplake.create("mem://documents")

ds.add_column("text", "text")

ds.commit()

ds = deeplake.create("mem://dataset")

ds.add_column("train_split", "text")

ds.add_column("confidence", "float32")

ds.add_column("label", "text")

ds.commit()

ds = deeplake.create("mem://images")

ds.add_column("id", "int32")

ds.add_column("image", "int32")

ds.add_column("embedding", deeplake.types.Array("float32", 1))

ds.commit()

ds = deeplake.create("mem://metadata")

ds.add_column("image_id", "int32")

ds.add_column("labels", "text")

ds.add_column("metadata", "text")

ds.add_column("verified", "bool")

ds.commit()

```

-->

Examples:

Vector similarity search:

```python

# Find similar embeddings

similar = deeplake.query('''

SELECT * FROM "mem://embeddings"

ORDER BY COSINE_SIMILARITY(vector, ARRAY[0.1, 0.2, 0.3]) DESC

LIMIT 100

''')

# Use results in training

dataloader = similar.pytorch()

```

Text semantic search:

```python

# Search documents using BM25

relevant = deeplake.query('''

SELECT * FROM "mem://documents"

ORDER BY BM25_SIMILARITY(text, 'machine learning') DESC

LIMIT 10

''')

```

Complex filtering:

```python

# Filter training data

train = deeplake.query('''

SELECT * FROM "mem://dataset"

WHERE "train_split" = 'train'

AND confidence > 0.9

AND label IN ('cat', 'dog')

''')

```

Joins for feature engineering:

```python

# Combine image features with metadata

features = deeplake.query('''

SELECT i.image, i.embedding, m.labels, m.metadata

FROM "mem://images" AS i

JOIN "mem://metadata" AS m ON i.id = m.image_id

WHERE m.verified = true

''')

```

"""

...

def query_async(query: str, token: str | None = None, creds: dict[str, str] | None = None) -> Future[DatasetView]:

"""

Asynchronously executes TQL queries optimized for ML data filtering and search.

Non-blocking version of `query()` for better performance with large datasets.

Supports the same TQL features including vector similarity search, text search,

filtering, and joins.

Args:

query: TQL query string supporting:

- Vector similarity: COSINE_SIMILARITY, EUCLIDEAN_DISTANCE

- Text search: BM25_SIMILARITY, CONTAINS

- Filtering: WHERE clauses

- Sorting: ORDER BY

- Joins: JOIN across datasets

token: Optional Activeloop authentication token

creds (dict, optional): Dictionary containing credentials used to access the dataset at the path.

- If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.

- It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.

- If nothing is given is, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets

Returns:

Future: Resolves to DatasetView that can be:

- Used directly in ML training

- Further filtered with additional queries

- Converted to PyTorch/TensorFlow dataloaders

- Materialized into a new dataset

<!-- test-context

```python

import deeplake

def prepare_training():

pass

```

-->

Examples:

Basic async query:

```python

# Run query asynchronously

future = deeplake.query_async('''

SELECT * FROM "mem://embeddings"

ORDER BY COSINE_SIMILARITY(vector, ARRAY[0.1, 0.2, 0.3]) DESC

''')

# Do other work while query runs

prepare_training()

# Get results when needed

results = future.result()

```

With async/await:

```python

async def search_similar():

results = await deeplake.query_async('''

SELECT * FROM "mem://images"

ORDER BY COSINE_SIMILARITY(embedding, ARRAY[0.1, 0.2, 0.3]) DESC

LIMIT 100

''')

return results

async def main():

similar = await search_similar()

```

Non-blocking check:

```python

future = deeplake.query_async(

"SELECT * FROM dataset WHERE train_split = 'train'"

)

if future.is_completed():

train_data = future.result()

else:

print("Query still running...")

```

"""

...

def explain_query(query: str, token: str | None = None, creds: dict[str, str] | None = None) -> ExplainQueryResult:

"""

Explains TQL query with optional authentication.

Args:

query: TQL query string to explain

token: Optional Activeloop authentication token

creds (dict, optional): Dictionary containing credentials used to access the dataset at the path.

Returns:

ExplainQueryResult: An explain result object to analyze the query.

<!-- test-context

```python

import deeplake

ds = deeplake.create("mem://explain_query")

ds.add_column("category", "text")

ds.append({"category": ["active", "inactive", "not sure"]})

ds.commit()

```

-->

Examples:

Explaining a query:

```python

explain_result = deeplake.explain_query('SELECT * FROM "mem://explain_query" WHERE category == \'active\'')

print(explain_result)

```

"""

...

class Client:

"""

Client for connecting to Activeloop services.

Handles authentication and API communication.

"""

endpoint: str

class Random:

"""

A pseudorandom number generator class that allows for deterministic random number generation

through seed control.

"""

seed: int | None

class Branch:

"""

Describes a branch within the dataset.

Branches are created using [deeplake.Dataset.branch][].

"""

__hash__: typing.ClassVar[None] = None

def __eq__(self, other: Branch) -> bool:

...

def __str__(self) -> str:

...

def delete(self) -> None:

"""

Deletes the branch from the dataset

"""

...

def open(self) -> Dataset:

"""

Opens corresponding branch of the dataset

"""

...

def open_async(self) -> Future[Dataset]:

"""

Asynchronously fetches the dataset corresponding to the branch and returns a Future object.

"""

...

def rename(self, new_name: str) -> None:

"""

Renames the branch within the dataset

"""

...

@property

def base(self) -> tuple[str, str] | None:

"""

The base branch id and version

"""

...

@property

def id(self) -> str:

"""

The unique identifier of the branch

"""

...

@property

def name(self) -> str:

"""

The name of the branch

"""

...

@property

def timestamp(self) -> datetime.datetime:

"""

The branch creation timestamp

"""

...

@property

def is_link(self) -> bool:

...

@property

def link_target(self) -> str | None:

...

class BranchView:

"""

Describes a read-only branch within the dataset.

"""

__hash__: typing.ClassVar[None] = None

def __eq__(self, other: BranchView) -> bool:

...

def __str__(self) -> str:

...

@property

def base(self) -> tuple[str, str] | None:

"""

The base branch id and version

"""

...

@property

def id(self) -> str:

"""

The unique identifier of the branch

"""

...

@property

def name(self) -> str:

"""

The name of the branch

"""

...

@property

def timestamp(self) -> datetime.datetime:

"""

The branch creation timestamp

"""

...

def open(self) -> ReadOnlyDataset:

"""

Opens corresponding branch of the dataset

"""

...