# type: ignore
from unittest import skip
import numpy as np
from time import time
import json
from tqdm import tqdm  # type: ignore
import posixpath

from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union

import hub
from hub.util.invalid_view_op import invalid_view_op
import numpy as np
from hub.api.info import load_info
from hub.client.log import logger
from hub.constants import FIRST_COMMIT_ID
from hub.constants import DEFAULT_MEMORY_CACHE_SIZE, DEFAULT_LOCAL_CACHE_SIZE, MB
from hub.core.fast_forwarding import ffw_dataset_meta
from hub.core.index import Index
from hub.core.lock import lock_dataset, unlock_dataset, Lock
from hub.core.meta.dataset_meta import DatasetMeta
from hub.core.storage import (
    LRUCache,
    S3Provider,
    GCSProvider,
    MemoryProvider,
)
from hub.core.tensor import Tensor, create_tensor, delete_tensor

from hub.core.version_control.commit_node import CommitNode  # type: ignore
from hub.core.version_control.dataset_diff import load_dataset_diff
from hub.htype import (
    HTYPE_CONFIGURATIONS,
    UNSPECIFIED,
    DEFAULT_HTYPE,
    verify_htype_key_value,
)
from hub.integrations import dataset_to_tensorflow
from hub.util.bugout_reporter import hub_reporter
from hub.util.dataset import try_flushing
from hub.util.cache_chain import generate_chain
from hub.util.hash import hash_inputs
from hub.util.htype import parse_sequence_htype
from hub.util.warnings import always_warn
from hub.util.exceptions import (
    CouldNotCreateNewDatasetException,
    InvalidKeyTypeError,
    InvalidTensorGroupNameError,
    InvalidTensorNameError,
    TensorMetaInvalidHtype,
    LockedException,
    MemoryDatasetCanNotBePickledError,
    PathNotEmptyException,
    TensorAlreadyExistsError,
    TensorDoesNotExistError,
    TensorGroupDoesNotExistError,
    InvalidTensorNameError,
    InvalidTensorGroupNameError,
    LockedException,
    TensorGroupAlreadyExistsError,
    ReadOnlyModeError,
    NotLoggedInError,
    EmptyCommitError,
)
from hub.util.keys import (
    dataset_exists,
    get_dataset_info_key,
    get_dataset_meta_key,
    tensor_exists,
    get_queries_key,
    get_queries_lock_key,
)
from hub.util.path import get_path_from_storage
from hub.util.remove_cache import get_base_storage
from hub.util.diff import get_all_changes_string, get_changes_and_messages
from hub.util.version_control import (
    auto_checkout,
    checkout,
    commit,
    current_commit_has_change,
    load_meta,
    warn_node_checkout,
    load_version_info,
)
from hub.client.utils import get_user_name


_LOCKABLE_STORAGES = {S3Provider, GCSProvider}


class Dataset:
    def __init__(
        self,
        storage: LRUCache,
        index: Optional[Index] = None,
        group_index: str = "",
        read_only: bool = False,
        public: Optional[bool] = False,
        token: Optional[str] = None,
        verbose: bool = True,
        version_state: Optional[Dict[str, Any]] = None,
        path: Optional[str] = None,
        is_iteration: bool = False,
        **kwargs,
    ):
        """Initializes a new or existing dataset.

        Args:
            storage (LRUCache): The storage provider used to access the dataset.
            index (Index, optional): The Index object restricting the view of this dataset's tensors.
            group_index (str): Name of the group this dataset instance represents.
            read_only (bool): Opens dataset in read only mode if this is passed as True. Defaults to False.
                Datasets stored on Hub cloud that your account does not have write access to will automatically open in read mode.
            public (bool, optional): Applied only if storage is Hub cloud storage and a new Dataset is being created. Defines if the dataset will have public access.
            token (str, optional): Activeloop token, used for fetching credentials for Hub datasets. This is optional, tokens are normally autogenerated.
            verbose (bool): If True, logs will be printed. Defaults to True.
            version_state (Dict[str, Any], optional): The version state of the dataset, includes commit_id, commit_node, branch, branch_commit_map and commit_node_map.
            is_iteration (bool): If this Dataset is being used as an iterator.
            **kwargs: Passing subclass variables through without errors.
            path: The path to the dataset.


        Raises:
            ValueError: If an existing local path is given, it must be a directory.
            ImproperDatasetInitialization: Exactly one argument out of 'path' and 'storage' needs to be specified.
                This is raised if none of them are specified or more than one are specifed.
            InvalidHubPathException: If a Hub cloud path (path starting with hub://) is specified and it isn't of the form hub://username/datasetname.
            AuthorizationException: If a Hub cloud path (path starting with hub://) is specified and the user doesn't have access to the dataset.
            PathNotEmptyException: If the path to the dataset doesn't contain a Hub dataset and is also not empty.
        """
        d: Dict[str, Any] = {}
        d["_client"] = d["org_id"] = d["ds_name"] = None
        # uniquely identifies dataset
        d["path"] = path or get_path_from_storage(storage)
        d["storage"] = storage
        d["_read_only"] = read_only
        d["_locked_out"] = False  # User requested write access but was denied
        d["is_iteration"] = is_iteration
        d["is_first_load"] = version_state is None
        d["index"] = index or Index()
        d["group_index"] = group_index
        d["_token"] = token
        d["public"] = public
        d["verbose"] = verbose
        d["version_state"] = version_state or {}
        d["_info"] = None
        d["_ds_diff"] = None
        self.__dict__.update(d)
        self._set_derived_attributes()
        self._first_load_init()
        self._initial_autoflush: List[
            bool
        ] = []  # This is a stack to support nested with contexts
        self._is_filtered_view = False
        self._view_info = None

    def _lock_lost_handler(self):
        """This is called when lock is acquired but lost later on due to slow update."""
        self.read_only = True
        always_warn(
            "Unable to update dataset lock as another machine has locked it for writing. Switching to read only mode."
        )
        self._locked_out = True

    def __enter__(self):
        self._initial_autoflush.append(self.storage.autoflush)
        self.storage.autoflush = False
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.storage.autoflush = self._initial_autoflush.pop()
        if not self._read_only:
            self.storage.maybe_flush()

    @property
    def num_samples(self) -> int:
        """Returns the length of the smallest tensor.
        Ignores any applied indexing and returns the total length.
        """
        return min(map(len, self.version_state["full_tensors"].values()), default=0)

    @property
    def meta(self) -> DatasetMeta:
        """Returns the metadata of the dataset."""
        return self.version_state["meta"]

    def __len__(self):
        """Returns the length of the smallest tensor"""
        tensor_lengths = [len(tensor) for tensor in self.tensors.values()]
        return min(tensor_lengths, default=0)

    def __getstate__(self) -> Dict[str, Any]:
        """Returns a dict that can be pickled and used to restore this dataset.

        Note:
            Pickling a dataset does not copy the dataset, it only saves attributes that can be used to restore the dataset.
            If you pickle a local dataset and try to access it on a machine that does not have the data present, the dataset will not work.
        """
        if self.path.startswith("mem://"):
            raise MemoryDatasetCanNotBePickledError
        keys = [
            "path",
            "_read_only",
            "index",
            "group_index",
            "public",
            "storage",
            "_token",
            "verbose",
            "version_state",
            "org_id",
            "ds_name",
            "_is_filtered_view",
            "_view_info",
        ]
        state = {k: getattr(self, k) for k in keys}
        return state

    def __setstate__(self, state: Dict[str, Any]):
        """Restores dataset from a pickled state.

        Args:
            state (dict): The pickled state used to restore the dataset.
        """
        state["is_first_load"] = True
        state["_info"] = None
        state["is_iteration"] = False
        self.__dict__.update(state)

        # clear cache while restoring
        self.storage.clear_cache_without_flush()
        self._initial_autoflush = []
        self.is_first_load = True
        self._info = None
        self._ds_diff = None
        self._set_derived_attributes()

    def __getitem__(
        self,
        item: Union[
            str, int, slice, List[int], Tuple[Union[int, slice, Tuple[int]]], Index
        ],
        is_iteration: bool = False,
    ):
        if isinstance(item, str):
            fullpath = posixpath.join(self.group_index, item)
            tensor = self._get_tensor_from_root(fullpath)
            if tensor is not None:
                return tensor[self.index]
            elif self._has_group_in_root(fullpath):
                return self.__class__(
                    storage=self.storage,
                    index=self.index,
                    group_index=posixpath.join(self.group_index, item),
                    read_only=self.read_only,
                    token=self._token,
                    verbose=False,
                    version_state=self.version_state,
                    path=self.path,
                )
            elif "/" in item:
                splt = posixpath.split(item)
                return self[splt[0]][splt[1]]
            else:
                raise TensorDoesNotExistError(item)
        elif isinstance(item, (int, slice, list, tuple, Index)):
            return self.__class__(
                storage=self.storage,
                index=self.index[item],
                group_index=self.group_index,
                read_only=self._read_only,
                token=self._token,
                verbose=False,
                version_state=self.version_state,
                path=self.path,
                is_iteration=is_iteration,
            )
        else:
            raise InvalidKeyTypeError(item)

    @invalid_view_op
    @hub_reporter.record_call
    def create_tensor(
        self,
        name: str,
        htype: str = UNSPECIFIED,
        dtype: Union[str, np.dtype] = UNSPECIFIED,
        sample_compression: str = UNSPECIFIED,
        chunk_compression: str = UNSPECIFIED,
        hidden: bool = False,
        **kwargs,
    ):
        """Creates a new tensor in the dataset.

        Args:
            name (str): The name of the tensor to be created.
            htype (str): The class of data for the tensor.
                The defaults for other parameters are determined in terms of this value.
                For example, `htype="image"` would have `dtype` default to `uint8`.
                These defaults can be overridden by explicitly passing any of the other parameters to this function.
                May also modify the defaults for other parameters.
            dtype (str): Optionally override this tensor's `dtype`. All subsequent samples are required to have this `dtype`.
            sample_compression (str): All samples will be compressed in the provided format. If `None`, samples are uncompressed.
            chunk_compression (str): All chunks will be compressed in the provided format. If `None`, chunks are uncompressed.
            **kwargs: `htype` defaults can be overridden by passing any of the compatible parameters.
                To see all `htype`s and their correspondent arguments, check out `hub/htypes.py`.
            hidden (bool): If True, the tensor will be hidden from ds.tensors but can still be accessed via ds[tensor_name]

        Returns:
            The new tensor, which can also be accessed by `self[name]`.

        Raises:
            TensorAlreadyExistsError: Duplicate tensors are not allowed.
            TensorGroupAlreadyExistsError: Duplicate tensor groups are not allowed.
            InvalidTensorNameError: If `name` is in dataset attributes.
            NotImplementedError: If trying to override `chunk_compression`.
            TensorMetaInvalidHtype: If invalid htype is specified.
            ValueError: If an illegal argument is specified.
        """
        # if not the head node, checkout to an auto branch that is newly created
        auto_checkout(self)
        name = name.strip("/")

        while "//" in name:
            name = name.replace("//", "/")

        full_path = posixpath.join(self.group_index, name)

        if tensor_exists(full_path, self.storage, self.version_state["commit_id"]):
            raise TensorAlreadyExistsError(name)

        if full_path in self._groups:
            raise TensorGroupAlreadyExistsError(name)

        if not name or name in dir(self):
            raise InvalidTensorNameError(name)

        is_sequence, htype = parse_sequence_htype(htype)
        kwargs["is_sequence"] = is_sequence

        if not self._is_root():
            return self.root.create_tensor(
                full_path, htype, dtype, sample_compression, chunk_compression, **kwargs
            )

        if "/" in name:
            self._create_group(posixpath.split(name)[0])

        # Seperate meta and info

        htype_config = HTYPE_CONFIGURATIONS.get(htype, {})
        info_keys = htype_config.copy().pop("_info", [])
        info_kwargs = {}
        meta_kwargs = {}
        for k, v in kwargs.items():
            if k in info_keys:
                verify_htype_key_value(htype, k, v)
                info_kwargs[k] = v
            else:
                meta_kwargs[k] = v

        # Set defaults
        for k in info_keys:
            if k not in info_kwargs:
                info_kwargs[k] = htype_config[k]

        create_tensor(
            name,
            self.storage,
            htype=htype,
            dtype=dtype,
            sample_compression=sample_compression,
            chunk_compression=chunk_compression,
            version_state=self.version_state,
            hidden=hidden,
            **meta_kwargs,
        )
        meta: DatasetMeta = self.meta
        ffw_dataset_meta(meta)
        meta.add_tensor(name, hidden=hidden)
        tensor = Tensor(name, self)  # type: ignore
        self.version_state["full_tensors"][name] = tensor
        if info_kwargs:
            tensor.info.update(info_kwargs)
        self.storage.maybe_flush()
        return tensor

    def _hide_tensor(self, tensor: str):
        self._tensors()[tensor].meta.set_hidden(True)
        self.meta._hide_tensor(tensor)
        self.storage.maybe_flush()

    @invalid_view_op
    @hub_reporter.record_call
    def delete_tensor(self, name: str, large_ok: bool = False):
        """Delete a tensor from the dataset.

        Args:
            name (str): The name of tensor to be deleted.
            large_ok (bool): Delete tensors larger than 1GB. Disabled by default.

        Returns:
            None

        Raises:
            TensorDoesNotExistError: If tensor of name `name` does not exist in the dataset.
            InvalidTensorNameError: If `name` is in dataset attributes.
        """
        auto_checkout(self)
        name = name.strip("/")

        while "//" in name:
            name = name.replace("//", "/")

        full_path = posixpath.join(self.group_index, name)

        if not tensor_exists(full_path, self.storage, self.version_state["commit_id"]):
            raise TensorDoesNotExistError(name)

        if not name or name in dir(self):
            raise InvalidTensorNameError(name)

        if not self._is_root():
            return self.root.delete_tensor(full_path, large_ok)

        if not large_ok:
            chunk_engine = self.version_state["full_tensors"][name].chunk_engine
            size_approx = chunk_engine.num_samples * chunk_engine.min_chunk_size
            if size_approx > hub.constants.DELETE_SAFETY_SIZE:
                logger.info(
                    f"Tensor {name} was too large to delete. Try again with large_ok=True."
                )
                return

        with self:
            delete_tensor(name, self)
            meta_key = get_dataset_meta_key(self.version_state["commit_id"])
            meta: DatasetMeta = self.storage.get_hub_object(meta_key, DatasetMeta)
            ffw_dataset_meta(meta)
            meta.delete_tensor(name)
            self.version_state["meta"] = meta
            self.version_state["full_tensors"].pop(name)

        self.storage.maybe_flush()

    @invalid_view_op
    @hub_reporter.record_call
    def delete_group(self, name: str, large_ok: bool = False):
        """Delete a tensor group from the dataset.

        Args:
            name (str): The name of tensor group to be deleted.
            large_ok (bool): Delete tensor groups larger than 1GB. Disabled by default.

        Returns:
            None

        Raises:
            TensorGroupDoesNotExistError: If tensor group of name `name` does not exist in the dataset.
            InvalidTensorGroupNameError: If `name` is in dataset attributes.
        """
        auto_checkout(self)
        name = name.strip("/")

        while "//" in name:
            name = name.replace("//", "/")

        full_path = posixpath.join(self.group_index, name)

        if full_path not in self._groups:
            raise TensorGroupDoesNotExistError(name)

        if not name or name in dir(self):
            raise InvalidTensorGroupNameError(name)

        if not self._is_root():
            return self.root.delete_group(full_path, large_ok)

        if not large_ok:
            size_approx = self[name].size_approx()
            if size_approx > hub.constants.DELETE_SAFETY_SIZE:
                logger.info(
                    f"Group {name} was too large to delete. Try again with large_ok=True."
                )
                return

        with self:
            meta = self.version_state["meta"]
            ffw_dataset_meta(meta)
            tensors = [
                posixpath.join(name, tensor)
                for tensor in self[name]._all_tensors_filtered(include_hidden=True)
            ]
            meta.delete_group(name)
            for tensor in tensors:
                delete_tensor(tensor, self)
                self.version_state["full_tensors"].pop(tensor)

        self.storage.maybe_flush()

    @invalid_view_op
    @hub_reporter.record_call
    def create_tensor_like(self, name: str, source: "Tensor") -> "Tensor":
        """Copies the `source` tensor's meta information and creates a new tensor with it. No samples are copied, only the meta/info for the tensor is.

        Args:
            name (str): Name for the new tensor.
            source (Tensor): Tensor who's meta/info will be copied. May or may not be contained in the same dataset.

        Returns:
            Tensor: New Tensor object.
        """

        info = source.info.__getstate__().copy()
        meta = source.meta.__getstate__().copy()
        del meta["min_shape"]
        del meta["max_shape"]
        del meta["length"]
        del meta["version"]

        destination_tensor = self.create_tensor(name, **meta)
        destination_tensor.info.update(info)
        return destination_tensor

    __getattr__ = __getitem__

    def __setattr__(self, name: str, value):
        if isinstance(value, (np.ndarray, np.generic)):
            raise TypeError(
                "Setting tensor attributes directly is not supported. To add a tensor, use the `create_tensor` method."
                + "To add data to a tensor, use the `append` and `extend` methods."
            )
        else:
            return super().__setattr__(name, value)

    def __iter__(self):
        for i in range(len(self)):
            yield self.__getitem__(i, is_iteration=True)

    def _load_version_info(self):
        """Loads data from version_control_file otherwise assume it doesn't exist and load all empty"""
        if self.version_state:
            return

        branch = "main"
        version_state = {"branch": branch}
        try:
            version_info = load_version_info(self.storage)
            version_state["branch_commit_map"] = version_info["branch_commit_map"]
            version_state["commit_node_map"] = version_info["commit_node_map"]
            commit_id = version_state["branch_commit_map"][branch]
            version_state["commit_id"] = commit_id
            version_state["commit_node"] = version_state["commit_node_map"][commit_id]
        except Exception:
            version_state["branch_commit_map"] = {}
            version_state["commit_node_map"] = {}
            # used to identify that this is the first commit so its data will not be in similar directory structure to the rest
            commit_id = FIRST_COMMIT_ID
            commit_node = CommitNode(branch, commit_id)
            version_state["commit_id"] = commit_id
            version_state["commit_node"] = commit_node
            version_state["branch_commit_map"][branch] = commit_id
            version_state["commit_node_map"][commit_id] = commit_node
        # keeps track of the full unindexed tensors
        version_state["full_tensors"] = {}
        self.__dict__["version_state"] = version_state

    def _lock(self, err=False):
        storage = get_base_storage(self.storage)

        if isinstance(storage, tuple(_LOCKABLE_STORAGES)) and (
            not self.read_only or self._locked_out
        ):
            try:
                # temporarily disable read only on base storage, to try to acquire lock, if exception, it will be again made readonly
                storage.disable_readonly()
                lock_dataset(
                    self,
                    lock_lost_callback=self._lock_lost_handler,
                )
            except LockedException as e:
                self.read_only = True
                self.__dict__["_locked_out"] = True
                if err:
                    raise e
                always_warn(
                    "Checking out dataset in read only mode as another machine has locked this version for writing."
                )
                return False
        return True

    def _unlock(self):
        unlock_dataset(self)

    def __del__(self):
        try:
            self._unlock()
        except Exception:  # python shutting down
            pass

    def commit(self, message: Optional[str] = None, allow_empty=False) -> str:
        """Stores a snapshot of the current state of the dataset.
        Note: Commiting from a non-head node in any branch, will lead to an auto checkout to a new branch.
        This same behaviour will happen if new samples are added or existing samples are updated from a non-head node.

        Args:
            message (str, optional): Used to describe the commit.
            allow_empty (bool): If True, commit even if there are no changes

        Returns:
            str: the commit id of the stored commit that can be used to access the snapshot.

        Raises:
            Exception: if dataset is a filtered view.
            EmptyCommitError: if there are no changes and user does not forced to commit unchanged data
        """
        if not allow_empty and not self.has_head_changes:
            raise EmptyCommitError(
                "There are no changes, commit is not done. Try again with allow_empty=True."
            )

        return self._commit(message)

    def _commit(self, message: Optional[str] = None, hash: Optional[str] = None) -> str:
        if self._is_filtered_view:
            raise Exception(
                "Cannot perform version control operations on a filtered dataset view."
            )

        try_flushing(self)

        self._initial_autoflush.append(self.storage.autoflush)
        self.storage.autoflush = False
        try:
            self._unlock()
            commit(self, message, hash)
            self._lock()
        finally:
            self.storage.autoflush = self._initial_autoflush.pop()
        self._info = None
        self._ds_diff = None

        # do not store commit message
        hub_reporter.feature_report(feature_name="commit", parameters={})

        return self.commit_id  # type: ignore

    def checkout(self, address: str, create: bool = False) -> Optional[str]:
        """Checks out to a specific commit_id or branch. If create = True, creates a new branch with name as address.
        Note: Checkout from a head node in any branch that contains uncommitted data will lead to an auto commit before the checkout.

        Args:
            address (str): The commit_id or branch to checkout to.
            create (bool): If True, creates a new branch with name as address.

        Returns:
            str: The commit_id of the dataset after checkout.

        Raises:
            Exception: if dataset is a filtered view.
        """
        return self._checkout(address, create)

    def _checkout(
        self, address: str, create: bool = False, hash: Optional[str] = None
    ) -> Optional[str]:
        if self._is_filtered_view:
            raise Exception(
                "Cannot perform version control operations on a filtered dataset view."
            )

        try_flushing(self)

        self._initial_autoflush.append(self.storage.autoflush)
        self.storage.autoflush = False
        try:
            self._unlock()
            checkout(self, address, create, hash)
            self._lock()
        finally:
            self.storage.autoflush = self._initial_autoflush.pop()
        self._info = None
        self._ds_diff = None

        # do not store address
        hub_reporter.feature_report(
            feature_name="checkout", parameters={"Create": str(create)}
        )
        commit_node = self.version_state["commit_node"]
        if self.verbose:
            warn_node_checkout(commit_node, create)

        return self.commit_id

    def log(self):
        """Displays the details of all the past commits."""
        commit_node = self.version_state["commit_node"]
        print("---------------\nHub Version Log\n---------------\n")
        print(f"Current Branch: {self.version_state['branch']}")
        if self.has_head_changes:
            print("** There are uncommitted changes on this branch.")
        print()
        while commit_node:
            if not commit_node.is_head_node:
                print(f"{commit_node}\n")
            commit_node = commit_node.parent

    def diff(
        self, id_1: Optional[str] = None, id_2: Optional[str] = None, as_dict=False
    ) -> Optional[Union[Dict, Tuple[Dict, Dict]]]:
        """Returns/displays the differences between commits/branches.
        For each tensor this contains information about the sample indexes that were added/modified as well as whether the tensor was created.

        Args:
            id_1 (str, optional): The first commit_id or branch name.
            id_2 (str, optional): The second commit_id or branch name.
            as_dict (bool, optional): If True, returns dictionares of the differences instead of printing them. Defaults to False.

        If both id_1 and id_2 are None, the differences between the current state and the previous commit will be calculated. If you're at the head of the branch, this will show the uncommitted changes, if any.
        If only id_1 is provided, the differences between the current state and id_1 will be calculated. If you're at the head of the branch, this will take into account the uncommitted changes, if any.
        If only id_2 is provided, a ValueError will be raised.
        If both id_1 and id_2 are provided, the differences between id_1 and id_2 will be calculated.

        Returns:
            Union[Dict, Tuple[Dict, Dict]]: The differences between the commits/branches if as_dict is True.
                If id_1 and id_2 are None, a single dictionary containing the differences between the current state and the previous commit will be returned.
                If only id_1 is provided, two dictionaries containing the differences in the current state and id_1 respectively will be returned.
                If only id_2 is provided, a ValueError will be raised.
                If both id_1 and id_2 are provided, two dictionaries containing the differences in id_1 and id_2 respectively will be returned.
            None: If as_dict is False.

            Example of a dict returned:
            {
                "image": {"data_added": [3, 6], "data_updated": {0, 2}, "created": False, "info_updated": False, "data_transformed_in_place": False},
                "label": {"data_added": [0, 3], "data_updated": {}, "created": True, "info_updated": False, "data_transformed_in_place": False},
                "other/stuff" : {data_added: [3, 3], data_updated: {1, 2}, created: True, "info_updated": False, "data_transformed_in_place": False}
            }

            Here the data_adeded is a range of sample indexes that were added to the tensor.
            For example [3, 6] means that sample 3, 4 and 5 were added.
            Another example [3, 3] means that no samples were added as the range is empty

            data_updated on the other hand is a set of sample indexes that were updated.
            For example {0, 2} means that sample 0 and 2 were updated.

            created is a boolean that is True if the tensor was created.

            info_updated is a boolean that is True if the info of the tensor was updated.

            data_transformed_in_place is a boolean that is True if the data of the tensor was transformed in place.


        Raises:
            ValueError: If both id_1 is None and id_2 is not None.
        """
        version_state, storage = self.version_state, self.storage
        res = get_changes_and_messages(version_state, storage, id_1, id_2)
        if as_dict:
            tensor_changes_1 = res[2]
            tensor_changes_2 = res[3]
            if id_1 is None and id_2 is None:
                return tensor_changes_1
            return tensor_changes_1, tensor_changes_2
        all_changes = get_all_changes_string(*res)
        print(all_changes)
        return None

    def _populate_meta(self):
        """Populates the meta information for the dataset."""
        if dataset_exists(self.storage):
            if self.verbose:
                logger.info(f"{self.path} loaded successfully.")
            load_meta(self)

        elif not self.storage.empty():
            # dataset does not exist, but the path was not empty
            raise PathNotEmptyException

        else:
            if self.read_only:
                # cannot create a new dataset when in read_only mode.
                raise CouldNotCreateNewDatasetException(self.path)
            meta = DatasetMeta()
            key = get_dataset_meta_key(self.version_state["commit_id"])
            self.version_state["meta"] = meta
            self.storage.register_hub_object(key, meta)
            self._register_dataset()
            self.flush()

    def _register_dataset(self):
        """overridden in HubCloudDataset"""

    def _send_query_progress(self, *args, **kwargs):
        """overridden in HubCloudDataset"""

    def _send_compute_progress(self, *args, **kwargs):
        """overridden in HubCloudDataset"""

    def _send_pytorch_progress(self, *args, **kwargs):
        """overridden in HubCloudDataset"""

    def _send_filter_progress(self, *args, **kwargs):
        """overridden in HubCloudDataset"""

    def _send_commit_event(self, *args, **kwargs):
        """overridden in HubCloudDataset"""

    def _send_dataset_creation_event(self, *args, **kwargs):
        """overridden in HubCloudDataset"""

    def _send_branch_creation_event(self, *args, **kwargs):
        """overridden in HubCloudDataset"""

    def _first_load_init(self):
        """overridden in HubCloudDataset"""

    @property
    def read_only(self):
        return self._read_only

    @property
    def has_head_changes(self):
        """Returns True if currently at head node and uncommitted changes are present."""
        commit_node = self.version_state["commit_node"]
        return not commit_node.children and current_commit_has_change(
            self.version_state, self.storage
        )

    def _set_read_only(self, value: bool, err: bool):
        storage = self.storage
        self.__dict__["_read_only"] = value
        if value:
            storage.enable_readonly()
            if isinstance(storage, LRUCache) and storage.next_storage is not None:
                storage.next_storage.enable_readonly()
        else:
            try:
                locked = self._lock(err=err)
                if locked:
                    self.storage.disable_readonly()
                    if (
                        isinstance(storage, LRUCache)
                        and storage.next_storage is not None
                    ):
                        storage.next_storage.disable_readonly()
                else:
                    self.__dict__["_read_only"] = True
            except LockedException as e:
                self.__dict__["_read_only"] = True
                raise e

    @read_only.setter
    @invalid_view_op
    def read_only(self, value: bool):
        self._set_read_only(value, True)

    @hub_reporter.record_call
    def pytorch(
        self,
        transform: Optional[Callable] = None,
        tensors: Optional[Sequence[str]] = None,
        num_workers: int = 1,
        batch_size: int = 1,
        drop_last: bool = False,
        collate_fn: Optional[Callable] = None,
        pin_memory: bool = False,
        shuffle: bool = False,
        buffer_size: int = 2048,
        use_local_cache: bool = False,
        use_progress_bar: bool = False,
    ):
        """Converts the dataset into a pytorch Dataloader.

        Note:
            Pytorch does not support uint16, uint32, uint64 dtypes. These are implicitly type casted to int32, int64 and int64 respectively.
            This spins up it's own workers to fetch data.

        Args:
            transform (Callable, optional) : Transformation function to be applied to each sample.
            tensors (List, optional): Optionally provide a list of tensor names in the ordering that your training script expects. For example, if you have a dataset that has "image" and "label" tensors, if `tensors=["image", "label"]`, your training script should expect each batch will be provided as a tuple of (image, label).
            num_workers (int): The number of workers to use for fetching data in parallel.
            batch_size (int): Number of samples per batch to load. Default value is 1.
            drop_last (bool): Set to True to drop the last incomplete batch, if the dataset size is not divisible by the batch size.
                If False and the size of dataset is not divisible by the batch size, then the last batch will be smaller. Default value is False.
                Read torch.utils.data.DataLoader docs for more details.
            collate_fn (Callable, optional): merges a list of samples to form a mini-batch of Tensor(s). Used when using batched loading from a map-style dataset.
                Read torch.utils.data.DataLoader docs for more details.
            pin_memory (bool): If True, the data loader will copy Tensors into CUDA pinned memory before returning them. Default value is False.
                Read torch.utils.data.DataLoader docs for more details.
            shuffle (bool): If True, the data loader will shuffle the data indices. Default value is False. Details about how hub shuffles data can be found at https://docs.activeloop.ai/how-hub-works/shuffling-in-ds.pytorch.
            buffer_size (int): The size of the buffer used to shuffle the data in MBs. Defaults to 2048 MB. Increasing the buffer_size will increase the extent of shuffling.
            use_local_cache (bool): If True, the data loader will use a local cache to store data. This is useful when the dataset can fit on the machine and we don't want to fetch the data multiple times for each iteration. Default value is False.
            use_progress_bar (bool): If True, tqdm will be wrapped around the returned dataloader. Default value is True.

        Returns:
            A torch.utils.data.DataLoader object.
        """
        from hub.integrations import dataset_to_pytorch as to_pytorch

        dataloader = to_pytorch(
            self,
            transform=transform,
            tensors=tensors,
            num_workers=num_workers,
            batch_size=batch_size,
            drop_last=drop_last,
            collate_fn=collate_fn,
            pin_memory=pin_memory,
            shuffle=shuffle,
            buffer_size=buffer_size,
            use_local_cache=use_local_cache,
        )

        if use_progress_bar:
            dataloader = tqdm(dataloader, desc=self.path, total=len(self) // batch_size)

        return dataloader

    @hub_reporter.record_call
    def filter(
        self,
        function: Union[Callable, str],
        num_workers: int = 0,
        scheduler: str = "threaded",
        progressbar: bool = True,
        store_result: bool = False,
        result_path: Optional[str] = None,
        result_ds_args: Optional[dict] = None,
    ):
        """Filters the dataset in accordance of filter function `f(x: sample) -> bool`

        Args:
            function(Callable | str): Filter function that takes sample as argument and returns True/False
                if sample should be included in result. Also supports simplified expression evaluations.
                See hub.core.query.DatasetQuery for more details.
            num_workers(int): Level of parallelization of filter evaluations.
                `0` indicates in-place for-loop evaluation, multiprocessing is used otherwise.
            scheduler(str): Scheduler to use for multiprocessing evaluation.
                `threaded` is default
            progressbar(bool): Display progress bar while filtering. True is default
            store_result (bool): If True, result of the filter will be saved to a dataset asynchronously.
            result_path (Optional, str): Path to save the filter result. Only applicable if `store_result` is True.
            result_ds_args (Optional, dict): Additional args for result dataset. Only applicable if `store_result` is True.

        Returns:
            View on Dataset with elements, that satisfy filter function


        Example:
            Following filters are identical and return dataset view where all the samples have label equals to 2.
            >>> dataset.filter(lambda sample: sample.labels.numpy() == 2)
            >>> dataset.filter('labels == 2')
        """
        from hub.core.query import filter_dataset, query_dataset
        from hub.core.query import DatasetQuery

        fn = query_dataset if isinstance(function, str) else filter_dataset
        return fn(
            self,
            function,
            num_workers=num_workers,
            scheduler=scheduler,
            progressbar=progressbar,
            store_result=store_result,
            result_path=result_path,
            result_ds_args=result_ds_args,
        )

    def _get_total_meta(self):
        """Returns tensor metas all together"""
        return {
            tensor_key: tensor_value.meta
            for tensor_key, tensor_value in self.version_state["full_tensors"].items()
        }

    def _set_derived_attributes(self):
        """Sets derived attributes during init and unpickling."""
        if self.is_first_load:
            self.storage.autoflush = True
            self._load_version_info()
            self._set_read_only(
                self._read_only, False
            )  # TODO: weird fix for dataset unpickling
            self._populate_meta()  # TODO: use the same scheme as `load_info`
        elif not self._read_only:
            self._lock()  # for ref counting
        if not self.is_iteration:
            self.index.validate(self.num_samples)

    @property
    def info(self):
        """Returns the information about the dataset."""
        if self._info is None:
            path = get_dataset_info_key(self.version_state["commit_id"])
            self.__dict__["_info"] = load_info(path, self)  # type: ignore
        return self._info

    @info.setter
    def info(self, value):
        if isinstance(value, dict):
            info = self.info
            info.replace_with(value)
        else:
            raise TypeError("Info must be set with type Dict")

    @property
    def _dataset_diff(self):
        if self._ds_diff is None:
            self.__dict__["_ds_diff"] = load_dataset_diff(self)
        return self._ds_diff

    @hub_reporter.record_call
    def tensorflow(self):
        """Converts the dataset into a tensorflow compatible format.

        See:
            https://www.tensorflow.org/api_docs/python/tf/data/Dataset

        Returns:
            tf.data.Dataset object that can be used for tensorflow training.
        """
        return dataset_to_tensorflow(self)

    def flush(self):
        """Necessary operation after writes if caches are being used.
        Writes all the dirty data from the cache layers (if any) to the underlying storage.
        Here dirty data corresponds to data that has been changed/assigned and but hasn't yet been sent to the
        underlying storage.
        """
        self.storage.flush()

    def clear_cache(self):
        """Flushes (see Dataset.flush documentation) the contents of the cache layers (if any) and then deletes contents
         of all the layers of it.
        This doesn't delete data from the actual storage.
        This is useful if you have multiple datasets with memory caches open, taking up too much RAM.
        Also useful when local cache is no longer needed for certain datasets and is taking up storage space.
        """
        if hasattr(self.storage, "clear_cache"):
            self.storage.clear_cache()

    def size_approx(self):
        """Estimates the size in bytes of the dataset.
        Includes only content, so will generally return an under-estimate.
        """
        tensors = self.version_state["full_tensors"].values()
        chunk_engines = [tensor.chunk_engine for tensor in tensors]
        size = sum(c.num_chunks * c.min_chunk_size for c in chunk_engines)
        for group in self._groups_filtered:
            size += self[group].size_approx()
        return size

    @invalid_view_op
    @hub_reporter.record_call
    def delete(self, large_ok=False):
        """Deletes the entire dataset from the cache layers (if any) and the underlying storage.
        This is an IRREVERSIBLE operation. Data once deleted can not be recovered.

        Args:
            large_ok (bool): Delete datasets larger than 1GB. Disabled by default.
        """

        if not large_ok:
            size = self.size_approx()
            if size > hub.constants.DELETE_SAFETY_SIZE:
                logger.info(
                    f"Hub Dataset {self.path} was too large to delete. Try again with large_ok=True."
                )
                return

        self._unlock()
        self.storage.clear()

    def __str__(self):
        path_str = ""
        if self.path:
            path_str = f"path='{self.path}', "

        mode_str = ""
        if self.read_only:
            mode_str = f"read_only=True, "

        index_str = f"index={self.index}, "
        if self.index.is_trivial():
            index_str = ""

        group_index_str = (
            f"group_index='{self.group_index}', " if self.group_index else ""
        )

        return f"Dataset({path_str}{mode_str}{index_str}{group_index_str}tensors={self.version_state['meta'].tensors})"

    __repr__ = __str__

    def _get_tensor_from_root(self, name: str) -> Optional[Tensor]:
        """Gets a tensor from the root dataset.
        Acesses storage only for the first call.
        """
        ret = self.version_state["full_tensors"].get(name)
        return ret

    def _has_group_in_root(self, name: str) -> bool:
        """Checks if a group exists in the root dataset.
        This is faster than checking `if group in self._groups:`
        """
        return name in self.version_state["meta"].groups

    @property
    def token(self):
        """Get attached token of the dataset"""

        return self._token

    @property
    def _ungrouped_tensors(self) -> Dict[str, Tensor]:
        """Top level tensors in this group that do not belong to any sub groups"""
        return {
            posixpath.basename(k): v
            for k, v in self.version_state["full_tensors"].items()
            if posixpath.dirname(k) == self.group_index
        }

    def _all_tensors_filtered(self, include_hidden: bool = True) -> List[str]:
        """Names of all tensors belonging to this group, including those within sub groups"""
        hidden_tensors = self.meta.hidden_tensors
        return [
            posixpath.relpath(t, self.group_index)
            for t in self.version_state["full_tensors"]
            if (not self.group_index or t.startswith(self.group_index + "/"))
            and (include_hidden or t not in hidden_tensors)
        ]

    def _tensors(self, include_hidden: bool = True) -> Dict[str, Tensor]:
        """All tensors belonging to this group, including those within sub groups. Always returns the sliced tensors."""
        return {
            t: self.version_state["full_tensors"][posixpath.join(self.group_index, t)][
                self.index
            ]
            for t in self._all_tensors_filtered(include_hidden)
        }

    @property
    def tensors(self) -> Dict[str, Tensor]:
        """All tensors belonging to this group, including those within sub groups. Always returns the sliced tensors."""
        return self._tensors(include_hidden=False)

    @property
    def branches(self):
        """Lists all the branches of the dataset.
        Returns:
            List of branches.
        """
        return list(self.version_state["branch_commit_map"])

    @property
    def commits(self) -> List[Dict]:
        """Lists all the commits leading to the current dataset state.
        Returns:
            List of dictionaries containing commit information.
        """
        commits = []
        commit_node = self.version_state["commit_node"]
        while commit_node:
            if not commit_node.is_head_node:
                commit_info = {
                    "commit": commit_node.commit_id,
                    "author": commit_node.commit_user_name,
                    "time": str(commit_node.commit_time)[:-7],
                    "message": commit_node.commit_message,
                }
                commits.append(commit_info)
            commit_node = commit_node.parent
        return commits

    def get_commit_details(self, commit_id) -> Dict:
        commit_node: CommitNode = self.version_state["commit_node_map"].get(commit_id)
        if commit_node is None:
            raise KeyError(f"Commit {commit_id} not found in dataset.")
        return {
            "commit": commit_node.commit_id,
            "author": commit_node.commit_user_name,
            "time": str(commit_node.commit_time)[:-7],
            "message": commit_node.commit_message,
        }

    @property
    def _groups(self) -> List[str]:
        """Names of all groups in the root dataset"""
        return self.meta.groups  # type: ignore

    @property
    def _groups_filtered(self) -> List[str]:
        """Names of all sub groups in this group"""
        groups_filtered = []
        for g in self._groups:
            dirname, basename = posixpath.split(g)
            if dirname == self.group_index:
                groups_filtered.append(basename)
        return groups_filtered

    @property
    def groups(self) -> Dict[str, "Dataset"]:
        """All sub groups in this group"""
        return {g: self[g] for g in self._groups_filtered}

    @property
    def commit_id(self) -> Optional[str]:
        """The lasted committed commit_id of the dataset. If there are no commits, this returns None."""
        commit_node = self.version_state["commit_node"]
        if not commit_node.is_head_node:
            return commit_node.commit_id

        parent = commit_node.parent

        if parent is None:
            return None
        else:
            return parent.commit_id

    @property
    def pending_commit_id(self) -> str:
        """The commit_id of the next commit that will be made to the dataset.
        If you're not at the head of the current branch, this will be the same as the commit_id.
        """
        return self.version_state["commit_id"]

    @property
    def branch(self) -> str:
        """The current branch of the dataset"""
        return self.version_state["branch"]

    def _is_root(self) -> bool:
        return not self.group_index

    @property
    def parent(self):
        """Returns the parent of this group. Returns None if this is the root dataset."""
        if self._is_root():
            return None
        autoflush = self.storage.autoflush
        ds = self.__class__(
            storage=self.storage,
            index=self.index,
            group_index=posixpath.dirname(self.group_index),
            read_only=self.read_only,
            public=self.public,
            token=self._token,
            verbose=self.verbose,
            version_state=self.version_state,
            path=self.path,
        )
        self.storage.autoflush = autoflush
        return ds

    @property
    def root(self):
        """Returns the root dataset of a group."""
        if self._is_root():
            return self
        autoflush = self.storage.autoflush
        ds = self.__class__(
            storage=self.storage,
            index=self.index,
            group_index="",
            read_only=self.read_only,
            public=self.public,
            token=self._token,
            verbose=self.verbose,
            version_state=self.version_state,
            path=self.path,
        )
        self.storage.autoflush = autoflush
        return ds

    def _create_group(self, name: str) -> "Dataset":
        """Internal method used by `create_group` and `create_tensor`."""
        meta: DatasetMeta = self.version_state["meta"]
        if not name or name in dir(self):
            raise InvalidTensorGroupNameError(name)
        fullname = name
        while name:
            if name in self.version_state["full_tensors"]:
                raise TensorAlreadyExistsError(name)
            meta.add_group(name)
            name, _ = posixpath.split(name)
        return self[fullname]

    def create_group(self, name: str) -> "Dataset":
        """Creates a tensor group. Intermediate groups in the path are also created."""
        if not self._is_root():
            return self.root.create_group(posixpath.join(self.group_index, name))
        name = name.strip("/")
        while "//" in name:
            name = name.replace("//", "/")
        if name in self._groups:
            raise TensorGroupAlreadyExistsError(name)
        return self._create_group(name)

    def rechunk(
        self,
        tensors: Optional[Union[str, List[str]]] = None,
        num_workers: int = 0,
        scheduler: str = "threaded",
        progressbar: bool = True,
    ):
        """Rewrites the underlying chunks to make their sizes optimal.
        This is usually needed in cases where a lot of updates have been made to the data.

        Args:
            tensors (str or list of str, optional): Name/names of the tensors to rechunk.
                If None, all tensors in the dataset are rechunked.
            num_workers (int): The number of workers to use for rechunking. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
            scheduler (str): The scheduler to be used for rechunking. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
                Defaults to 'threaded'.
            progressbar (bool): Displays a progress bar if True (default).
        """

        if tensors is None:
            tensors = list(self._tensors())
        elif isinstance(tensors, str):
            tensors = [tensors]

        # identity function that rechunks
        @hub.compute
        def rechunking(sample_in, samples_out):
            for tensor in tensors:
                samples_out[tensor].append(sample_in[tensor])

        rechunking().eval(
            self,
            num_workers=num_workers,
            scheduler=scheduler,
            progressbar=progressbar,
            skip_ok=True,
        )

    # the below methods are used by cloudpickle dumps
    def __origin__(self):
        return None

    def __values__(self):
        return None

    def __type__(self):
        return None

    def __union_params__(self):
        return None

    def __tuple_params__(self):
        return None

    def __result__(self):
        return None

    def __args__(self):
        return None

    def __bool__(self):
        return True

    def extend(self, samples: Dict[str, Any], skip_ok: bool = False):
        """Appends multiple rows of samples to mutliple tensors at once. This method expects all tensors being updated to be of the same length.
        Args:
            samples (Dict[str, Any]): Dictionary with tensor names as keys and samples as values.
            skip_ok (bool): Skip tensors not in `samples` if set to True.
        Raises:
            KeyError: If any tensor in the dataset is not a key in `samples` and `skip_ok` is False.
            TensorDoesNotExistError: If tensor in `samples` does not exist.
            ValueError: If all tensors being updated are not of the same length.
            NotImplementedError: If an error occurs while writing tiles.
            Exception: Error while attempting to rollback appends.
        """
        if isinstance(samples, Dataset):
            samples = samples.tensors
        if not samples:
            return
        n = len(samples[next(iter(samples.keys()))])
        for v in samples.values():
            if len(v) != n:
                sizes = {k: len(v) for (k, v) in samples.items()}
                raise ValueError(
                    f"Incoming samples are not of equal lengths. Incoming sample sizes: {sizes}"
                )
        for i in range(n):
            self.append({k: v[i] for k, v in samples.items()})

    def append(self, sample: Dict[str, Any], skip_ok: bool = False):
        """Append samples to mutliple tensors at once. This method expects all tensors being updated to be of the same length.
        Args:
            sample (dict): Dictionary with tensor names as keys and samples as values.
            skip_ok (bool): Skip tensors not in `sample` if set to True.
        Raises:
            KeyError: If any tensor in the dataset is not a key in `sample` and `skip_ok` is False.
            TensorDoesNotExistError: If tensor in `sample` does not exist.
            ValueError: If all tensors being updated are not of the same length.
            NotImplementedError: If an error occurs while writing tiles.
            Exception: Error while attempting to rollback appends.
        """
        if isinstance(sample, Dataset):
            sample = sample.tensors
        if not skip_ok:
            for k in self.tensors:
                if k not in sample:
                    raise KeyError(
                        f"Required tensor not provided: {k}. Use ds.append(sample, skip_ok=True) to skip tensors."
                    )
        for k in sample:
            if k not in self._tensors():
                raise TensorDoesNotExistError(k)
        if len(set(map(len, (self[k] for k in sample)))) != 1:
            raise ValueError(
                "When appending using Dataset.append, all tensors are expected to have the same length."
            )
        tensors_appended = []
        with self:
            for k, v in sample.items():
                try:
                    tensor = self[k]
                    enc = tensor.chunk_engine.chunk_id_encoder
                    num_chunks = enc.num_chunks
                    tensor.append(v)
                    tensors_appended.append(k)
                except Exception as e:
                    new_num_chunks = enc.num_chunks
                    num_chunks_added = new_num_chunks - num_chunks
                    if num_chunks_added > 1:
                        # This is unlikely to happen, i.e the sample passed the validation
                        # steps and tiling but some error occured while writing tiles to chunks
                        raise NotImplementedError(
                            "Unable to recover from error while writing tiles."
                        ) from e
                    elif num_chunks_added == 1:
                        enc._encoded = enc._encoded[:-1]
                    for k in tensors_appended:
                        try:
                            self[k]._pop()
                        except Exception as e2:
                            raise Exception(
                                "Error while attepting to rollback appends"
                            ) from e2
                    raise e

    def _view_hash(self) -> str:
        """Generates a unique hash for a filtered dataset view."""
        return hash_inputs(
            self.path,
            *[e.value for e in self.index.values],
            self.pending_commit_id,
            getattr(self, "_query", None),
        )

    def _get_view_info(self):
        if self._view_info is None:
            tm = getattr(self, "_created_at", time())
            hash = self._view_hash()
            info = {
                "id": hash,
                "description": "Virtual Datasource",
                "virtual-datasource": True,
                "source-dataset": self.path,
                "source-dataset-version": self.version_state["commit_id"],
                "created_at": tm,
            }

            query = getattr(self, "_query", None)
            if query:
                info["query"] = query
                info["source-dataset-index"] = getattr(self, "_source_ds_idx", None)
            self._view_info = info
        return self._view_info

    @staticmethod
    def _write_queries_json(ds, info):
        base_storage = get_base_storage(ds.storage)
        storage_read_only = base_storage.read_only
        if ds._locked_out:
            # Ignore storage level lock since we have file level lock
            base_storage.read_only = False
        lock = Lock(base_storage, get_queries_lock_key())
        lock.acquire(timeout=10, force=True)
        queries_key = get_queries_key()
        try:
            try:
                queries = json.loads(base_storage[queries_key].decode("utf-8"))
            except KeyError:
                queries = []
            queries.append(info)
            base_storage[queries_key] = json.dumps(queries).encode("utf-8")
        finally:
            lock.release()
            base_storage.read_only = storage_read_only

    def _write_vds(self, vds):
        """Writes the indices of this view to a vds."""
        info = self._get_view_info()
        with vds:
            vds.info.update(info)
            vds.create_tensor("VDS_INDEX", dtype="uint64").extend(
                list(self.index.values[0].indices(len(self)))
            )

    def _store_view_in_subdir(self):
        """Stores this view under ".queries" sub directory of same storage."""

        info = self._get_view_info()
        hash = info["id"]
        path = f".queries/{hash}"
        self.flush()
        get_base_storage(self.storage).subdir(path).clear()
        vds = self._sub_ds(path, empty=True)
        self._write_vds(vds)
        Dataset._write_queries_json(self, info)
        return vds

    def _store_view_in_user_queries_dataset(self):
        """Stores this view under hub://username/queries
        Only applicable for views of hub datasets.
        """
        if len(self.index.values) > 1:
            raise NotImplementedError("Storing sub-sample slices is not supported yet.")
        username = get_user_name()
        if username == "public":
            raise NotLoggedInError("Unable to save query result. Not logged in.")

        info = self._get_view_info()
        hash = info["id"]

        queries_ds_path = f"hub://{username}/queries"

        try:
            queries_ds = hub.dataset(
                queries_ds_path, verbose=False
            )  # create if doesn't exist
        except PathNotEmptyException:
            hub.delete(queries_ds_path, force=True)
            queries_ds = hub.dataset(queries_ds_path, verbose=False)

        queries_ds._unlock()  # we don't need locking as no data will be added to this ds.

        path = f"hub://{username}/queries/{hash}"

        vds = hub.empty(path, overwrite=True)

        self._write_vds(vds)

        Dataset._write_queries_json(queries_ds, info)

        return vds

    def _store_view_in_path(self, path, **ds_args):
        """Stores this view at a given dataset path"""
        vds = hub.dataset(path, **ds_args)
        self._write_vds(vds)
        return vds

    def store(self, path: Optional[str] = None, **ds_args) -> str:
        """Stores a dataset view as a virtual dataset (VDS)

        Args:
            path (Optional, str): If specified, the VDS will stored as a standalone dataset at the specified path. If not,
                the VDS is stored under `.queries` subdirectory of the source dataset's storage. If the user doesn't have
                write access to the source dataset and the source dataset is a hub cloud dataset, then the VDS is stored
                is stored under the user's hub account and can be accessed using hub.load(f"hub://{username}/queries/{query_hash}").
            ds_args (dict): Additional args for creating VDS when path is specified. (See documentation for `hub.dataset()`)

        Returns:
            (str) Path to the stored VDS.
        """
        return self._store(path, False, **ds_args)

    def _store(self, path: Optional[str] = None, _ret_ds: bool = False, **ds_args):
        """Stores a dataset view as a virtual dataset (VDS)

        Args:
            path (Optional, str): If specified, the VDS will stored as a standalone dataset at the specified path. If not,
                the VDS is stored under `.queries` subdirectory of the source dataset's storage. If the user doesn't have
                write access to the source dataset and the source dataset is a hub cloud dataset, then the VDS is stored
                is stored under the user's hub account and can be accessed using hub.load(f"hub://{username}/queries/{query_hash}").
            _ret_ds (bool): If True, the VDS is retured as such without converting it to a view. If False, the VDS path is returned.
                Default False.
            ds_args (dict): Additional args for creating VDS when path is specified. (See documentation for `hub.dataset()`)

        Returns:
            If _ret_ds is True, the VDS is returned, else path to the VDS is returned.

        Raises:
            NotImplementedError: When storing sub-sample slices and saving views inplace for in-memory datasets.
            ReadOnlyModeError: When attempting to save a view inplace and the user doesn't have write access.
        """
        if len(self.index.values) > 1:
            raise NotImplementedError("Storing sub-sample slices is not supported yet.")

        if path is None and hasattr(self, "_vds"):
            vds = self._vds
        elif path is None:
            if isinstance(self, MemoryProvider):
                raise NotImplementedError(
                    "Saving views inplace is not supported for in-memory datasets."
                )
            if self.read_only:
                if isinstance(self, hub.core.dataset.HubCloudDataset):
                    vds = self._store_view_in_user_queries_dataset()
                else:
                    raise ReadOnlyModeError(
                        "Cannot save view in read only dataset. Speicify a path to store the view in a different location."
                    )
            else:
                vds = self._store_view_in_subdir()
        else:
            vds = self._store_view_in_path(path, **ds_args)
        if _ret_ds:
            return vds
        return vds.path

    def _get_view(self):
        """Returns a view for this VDS. Only works if this Dataset is a virtual dataset.

        Returns:
            A view of the source dataset based on the indices from VDS.

        Raises:
            Exception: If this is not a VDS.
        """
        try:
            ds = hub.dataset(path=self.info["source-dataset"], verbose=False)
        except KeyError:
            raise Exception("Dataset._get_view() works only for virtual datasets.")
        ds = ds[self.VDS_INDEX.numpy().reshape(-1).tolist()]
        ds._vds = self
        return ds

    def _get_empty_vds(
        self, vds_path: Optional[str] = None, query: Optional[str] = None, **vds_args
    ):
        """Returns an empty VDS with this dataset as the source dataset. Internal.

        Args:
            vds_path (Optional, str): If specified, the vds will be stored at this path. Else the vds will be stored
                under `.queries` subdirectory.
            query (Optional, str): Query string associated with this view.
            vds_args (dict): Additional args for creating vds when path is specified.

        Returns:
            Empty VDS with this dataset as the source dataset.
        """
        view = self[:0]
        if query:
            view._query = query
        return view._store(vds_path, _ret_ds=True, **vds_args)

    def _get_query_history(self) -> List[str]:
        """
        Internal. Returns a list of hashes which can be passed to Dataset._get_stored_vds to get a dataset view.
        """
        try:
            queries = json.loads(self.storage[get_queries_key()].decode("utf-8"))
            return queries
        except KeyError:
            return []

    def _sub_ds(
        self,
        path,
        empty=False,
        memory_cache_size: int = DEFAULT_MEMORY_CACHE_SIZE,
        local_cache_size: int = DEFAULT_LOCAL_CACHE_SIZE,
    ):
        """Loads a nested dataset. Internal.
        Note: Virtual datasets are returned as such, they are not converted to views.

        Args:
            path (str): Path to sub directory
            empty (bool): If True, all contents of the sub directory is cleared before initializing the sub dataset.
            memory_cache_size (int): Memory cache size for the sub dataset.
            local_cache_size (int): Local storage cache size for the sub dataset.

        Returns:
            Sub dataset
        """
        base_storage = get_base_storage(self.storage)
        sub_storage = base_storage.subdir(path)

        if self.path.startswith("hub://"):
            path = posixpath.join(self.path, path)
            cls = hub.core.dataset.HubCloudDataset
        else:
            path = sub_storage.root
            cls = hub.core.dataset.Dataset

        return cls(
            generate_chain(
                sub_storage,
                memory_cache_size * MB,
                local_cache_size * MB,
            ),
            path=path,
            token=self._token,
        )

    def _get_stored_vds(self, hash: str):
        """Returns a vds stored under the `.queries` subdirectory given its hash.

        Args:
            hash (str): Hash of the required vds.

        Returns:
            VDS with the specified hash.
        """
        return self._get_sub_ds(".queries/" + hash)
