from amuse.support.core import CompositeDictionary
from amuse.support.core import compare_version_strings
from amuse.support import exceptions
from amuse.datamodel.base import *
from amuse.datamodel import base
from amuse.datamodel.memory_storage import *
from amuse.datamodel import trees
from amuse.units import constants
from amuse.units import units
from amuse.units import quantities
from amuse.units import trigo
from amuse.units.quantities import Quantity
from amuse.units.quantities import new_quantity
from amuse.units.quantities import is_quantity
from amuse.units.quantities import as_vector_quantity
from amuse.units.quantities import zero
from amuse.units.quantities import AdaptingVectorQuantity

import random
import numpy
from numpy import ma

try:
    from types import EllipsisType
except:
    EllipsisType = type(Ellipsis)

class AbstractParticleSet(AbstractSet):
    """
    Abstract superclass of all sets of particles.
    This class defines common code for all particle sets.

    Particle sets define dynamic attributes. Attributes
    can be set and retrieved on the particles using common python
    syntax. These attributes can only have values with units.

    >>> particles = Particles(2)
    >>> particles.mass = [10.0, 20.0] | units.kg
    >>> particles.mass
    quantity<[10.0, 20.0] kg>

    >>> particles.mass = 1.0 | units.kg
    >>> particles.mass
    quantity<[1.0, 1.0] kg>

    Particle sets can be iterated over.

    >>> particles = Particles(2)
    >>> particles.mass = [10.0, 20.0] | units.kg
    >>> for particle in particles:
    ...     print particle.mass
    ...
    10.0 kg
    20.0 kg


    Particle sets can be indexed.

    >>> particles = Particles(3)
    >>> particles.x = [10.0, 20.0, 30.0] | units.m
    >>> particles[1].x
    quantity<20.0 m>


    Particle sets can be sliced.

    >>> particles = Particles(3)
    >>> particles.x = [10.0, 20.0, 30.0] | units.m
    >>> particles[1:].x
    quantity<[20.0, 30.0] m>


    Particle sets can be copied.

    >>> particles = Particles(3)
    >>> particles.x = [10.0, 20.0, 30.0] | units.m
    >>> copy = particles.copy()
    >>> particles.x = 2.0 | units.m
    >>> particles.x
    quantity<[2.0, 2.0, 2.0] m>
    >>> copy.x
    quantity<[10.0, 20.0, 30.0] m>


    Particle sets can be added together.
    Attribute values are not stored by the resulting subset. The subset
    provides a view on two or more sets of particles. Changing attributes
    of the sum of sets will also change the attributes of each original
    set, contrary to copy().

    >>> particles = Particles(4)
    >>> particles1 = particles[:2]
    >>> particles1.x = [1.0, 2.0] | units.m
    >>> particles2 = particles[2:]
    >>> particles2.x = [3.0, 4.0] | units.m
    >>> new_set = particles1 + particles2
    >>> print len(new_set)
    4
    >>> print new_set.x
    [1.0, 2.0, 3.0, 4.0] m


    Particle sets can be subtracted from each other.
    Like with addition, attribute values are not stored by the resulting
    subset.

    >>> particles = Particles(4)
    >>> particles.x = [1.0, 2.0, 3.0, 4.0] | units.m
    >>> junk = particles[2:]
    >>> new_set = particles - junk
    >>> print len(new_set)
    2
    >>> print new_set.x
    [1.0, 2.0] m
    >>> print particles.x
    [1.0, 2.0, 3.0, 4.0] m


    Particle sets can have instance based or global vector attributes.
    A particle set stores a list of scalar values for each attribute.
    Some attributes are more naturally accessed as lists
    of vector values. Once defined, a particle set can
    convert the scalar values of 2 or more attributes into one
    vector attribute.

    >>> from amuse.support.data import particle_attributes
    >>> particles = Particles(2)
    >>> particles.x = [1.0 , 2.0] | units.m
    >>> particles.y = [3.0 , 4.0] | units.m
    >>> particles.z = [5.0 , 6.0] | units.m
    >>> particles.add_vector_attribute("p", ["x","y","z"])
    >>> particles.p
    quantity<[[1.0, 3.0, 5.0], [2.0, 4.0, 6.0]] m>
    >>> particles.p[0]
    quantity<[1.0, 3.0, 5.0] m>
    >>> particles.position # "position" is a global vector attribute, coupled to x,y,z
    quantity<[[1.0, 3.0, 5.0], [2.0, 4.0, 6.0]] m>

    """

    # this construct is needed to ensure that numpy see's grids
    # as objects and not as sequences
    # if we put a grid in a numpy object array we want the
    # grid in a field of that array and not the contents of
    # the grid (i.e. the grid points)
    # grids have the same trick
    if compare_version_strings(numpy.__version__, '1.7.0') < 0:
        __array_interface__ = {'shape':() }
    else:
        __array_interface__ = {'shape':(),'typestr':'|O4' }

    GLOBAL_DERIVED_ATTRIBUTES = {}


    def __init__(self, original = None):
        AbstractSet.__init__(self, original)


    def __delattr__(self, name):
        self.remove_attribute_from_store(name)

    #
    # Particle storage interface
    #

    def remove_attribute_from_store(self, name):
        pass
        
    def remove_particles_from_store(self, indices):
        pass

    def get_values_in_store(self, keys, attributes):
        pass

    def get_attribute_names_defined_in_store(self):
        return []

    def get_settable_attribute_names_defined_in_store(self):
        return []

    def get_indices_of_keys(self, keys):
        pass


    #
    #
    #

    def _values_of_particle(self, index):
        attributes = self.get_attribute_names_defined_in_store()
        values = self.get_values_in_store(numpy.asarray([index]), attributes)

        for attribute, val in zip(attributes, values):
            yield attribute, val[0]

    #
    # public API
    #
    def __iter__(self):
        original_set = self._original_set()
        for key, index in zip(self.get_all_keys_in_store(), self.get_all_indices_in_store()):
            yield original_set._get_particle_unsave(key, index)


    def get_all_particles_at(self, *indices):
        all_keys = self.get_all_keys_in_store()
        selectedkeyes = [all_keys[x] for x in indices]
        return self._subset(selectedkeyes)

    def __str__(self):
        """
        Display string of a particle set.

        >>> p0 = Particle(10)
        >>> p1 = Particle(11)
        >>> particles = Particles()
        >>> particles.add_particle(p0) # doctest:+ELLIPSIS
        <amuse.datamodel.particles.Particle object at ...>
        >>> particles.add_particle(p1) # doctest:+ELLIPSIS
        <amuse.datamodel.particles.Particle object at ...>
        >>> particles.x = [4.0 , 3.0] | units.m
        >>> particles.y = [5.0 , 2.0] | units.km
        >>> print particles
                         key            x            y
                           -            m           km
        ====================  ===========  ===========
                          10    4.000e+00    5.000e+00
                          11    3.000e+00    2.000e+00
        ====================  ===========  ===========

        """
        return self.to_string()


    def to_string(self, attributes_to_show = None, split_at = 20):
        """
        Display string of a particle set.

        >>> p0 = Particle(10)
        >>> p1 = Particle(11)
        >>> particles = Particles()
        >>> particles.add_particle(p0) # doctest:+ELLIPSIS
        <amuse.datamodel.particles.Particle object at ...>
        >>> particles.add_particle(p1) # doctest:+ELLIPSIS
        <amuse.datamodel.particles.Particle object at ...>
        >>> particles.x = [4.0 , 3.0] | units.m
        >>> particles.y = [5.0 , 2.0] | units.km
        >>> print particles.to_string()
                         key            x            y
                           -            m           km
        ====================  ===========  ===========
                          10    4.000e+00    5.000e+00
                          11    3.000e+00    2.000e+00
        ====================  ===========  ===========

        """
        attributes = sorted(self.get_attribute_names_defined_in_store())
        if attributes_to_show:
            attributes = [x for x in attributes if x in attributes_to_show]
        
        if len(self) == 0:
            return self.empty_particles_set_string(attributes)

        format_float = '{0: >11.3e}'.format
        format_str20 = '{0: >20}'.format
        format_str11 = '{0!s: >11}'.format

        columns = [[format_str11(x)] for x in attributes]
        columns.insert(0,[format_str20('key')])

        all_values = self.get_values_in_store(self.get_all_indices_in_store(), attributes)
        for index, quantity in enumerate(all_values):
            column = columns[index + 1]
            if hasattr(quantity, 'unit'):
                column.append(format_str11(str(quantity.unit)))
                quantity = quantity.number
            else:
                column.append(format_str11('none'))

            column.append('=' * 11)
            if len(quantity) > split_at * 2:
                if isinstance(quantity, LinkedArray):
                    values_to_show = list(map(format_str11,quantity[:split_at].to_print_list()))
                    values_to_show.append(format_str11('...'))
                    values_to_show.extend(map(format_str11,quantity[-split_at:].to_print_list()))
                elif hasattr(quantity, 'dtype'):
                    if numpy.issubdtype(quantity.dtype, numpy.floating):
                        values_to_show = list(map(format_float,quantity[:split_at]))
                        values_to_show.append(format_str11('...'))
                        values_to_show.extend(map(format_float,quantity[-split_at:]))
                    else:
                        values_to_show = list(map(format_str11,quantity[:split_at]))
                        values_to_show.append(format_str11('...'))
                        values_to_show.extend(map(format_str11,quantity[-split_at:]))
                else:
                    values_to_show = list(map(format_str11,quantity[:split_at]))
                    values_to_show.append(format_str11('...'))
                    values_to_show.extend(map(format_str11,quantity[-split_at:]))
            else:
                if isinstance(quantity, LinkedArray):
                    values_to_show = list(map(format_str11,quantity.to_print_list()))
                elif hasattr(quantity, 'dtype'):
                    if numpy.issubdtype(quantity.dtype, numpy.floating):
                        try:
                            values_to_show = list(map(format_float,quantity))
                        except ValueError:
                            values_to_show = list(map(format_str11,quantity))
                    else:
                        values_to_show = list(map(format_str11,quantity))
                else:
                    values_to_show = list(map(format_str11, quantity))

            column.extend(values_to_show)
            column.append('=' * 11)

        column = columns[0]
        column.append(format_str20("-"))
        column.append('=' * 20)
        particle_keys = self.get_all_keys_in_store()
        if len(particle_keys) > split_at * 2:
            values_to_show = list(map(format_str20, particle_keys[:split_at]))
            values_to_show.append(format_str20('...'))
            values_to_show.extend(map(format_str20, particle_keys[-split_at:]))
        else:
            values_to_show = list(map(format_str20,particle_keys))

        column.extend(values_to_show)

        column.append('=' * 20)

        rows = []
        for i in range(len(columns[0])):

            row = [x[i] for x in columns]
            rows.append(row)

        lines = ['  '.join(x) for x in rows]
        return '\n'.join(lines)


    def _get_particle(self, key):
        if self.has_key_in_store(key):
            return Particle(key, self._original_set())
        else:
            return None

    def _get_particle_unsave(self, key, index = None):
        return Particle(
            key,
            self._original_set(),
            set_index = index,
            set_version = self._get_version()
        )

    def can_extend_attributes(self):
        return self._original_set().can_extend_attributes()

    def add_attribute_domain(self, namespace):
        self._derived_attributes[namespace] = DomainAttribute(namespace)

    def _is_superset(self):
        return False

    def as_binary_tree(self, name_of_first_child = 'child1', name_of_second_child = 'child2'):
        return trees.ChildTreeOnParticleSet(self, (name_of_first_child, name_of_second_child))

    def new_binary_tree_wrapper(self, name_of_first_child = 'child1', name_of_second_child = 'child2'):
        return trees.ChildTreeOnParticleSet(self, (name_of_first_child, name_of_second_child))

    def copy(self, memento = None, keep_structure = False, filter_attributes = lambda particle_set, x : True):
        attributes = self.get_attribute_names_defined_in_store()
        attributes = [x for x in attributes if filter_attributes(self, x)]
        keys = self.get_all_keys_in_store()
        indices = self.get_all_indices_in_store()
        values = self.get_values_in_store(indices, attributes)
        result = self._factory_for_new_collection()()
        if memento is None:
            memento = {}
        memento[id(self._original_set())] = result

        converted = []
        for x in values:
            if isinstance(x, (LinkedArray, ParticlesSubset)):
                converted.append(x.copy(memento, keep_structure, filter_attributes))
            else:
                converted.append(x)
        result.add_particles_to_store(keys, attributes, converted)

        result._private.collection_attributes = self._private.collection_attributes._copy_for_collection(result)

        return result



    def copy_to_new_particles(self, keys = None, keys_generator = None, memento = None, keep_structure = False, filter_attributes = lambda particle_set, x : True):
        if keys_generator is None:
            keys_generator = base.UniqueKeyGenerator

        my_keys = self.get_all_keys_in_store()

        if not keys is None:
            if len(keys) != len(my_keys):
                    raise Exception('not enough new keys given for the copy')
            else:
                particle_keys = keys
        else:
            particle_keys = keys_generator.next_set_of_keys(len(my_keys))

        attributes = self.get_attribute_names_defined_in_store()
        attributes = [x for x in attributes if filter_attributes(self, x)]
        indices = self.get_all_indices_in_store()
        values = self.get_values_in_store(indices, attributes)
        result = self._factory_for_new_collection()()
        if memento is None:
            memento = {}

        converted = []
        for x in values:
            if isinstance(x, LinkedArray):
                converted.append(x.copy(memento, keep_structure, filter_attributes))
            else:
                converted.append(x)

        memento[id(self._original_set())] = result

        result.add_particles_to_store(particle_keys, attributes, converted)

        result._private.collection_attributes = self._private.collection_attributes._copy_for_collection(result)

        return result

    def _factory_for_new_collection(self):
        return Particles

    def empty_copy(self):
        """
        Creates a new in memory set and copies the particles to it.
        The attributes and values are not copied.The history
        of the set is not copied over.

        >>> from amuse.datamodel import Particles
        >>> from amuse.units import units
        >>> original = Particles(2)
        >>> original.mass = 0 | units.m
        >>> print hasattr(original, "mass")
        True
        >>> print len(original)
        2
        >>> copy = original.empty_copy()
        >>> print hasattr(copy, "mass")
        False
        >>> print len(copy)
        2

        """
        keys = self.get_all_keys_in_store()
        result = Particles()
        result.add_particles_to_store(keys, [],[])
        return result

    def copy_values_of_attribute_to(self, attribute_name, particles):
        """
        Copy values of one attribute from this set to the
        other set. Will only copy values for the particles
        in both sets. See also :meth:`synchronize_to`.

        If you need to do this a lot, setup a dedicated
        channel.

        >>> particles1 = Particles(2)
        >>> particles1.x = [1.0, 2.0] | units.m
        >>> particles2 = particles1.copy()
        >>> print particles2.x
        [1.0, 2.0] m
        >>> p3 = particles1.add_particle(Particle())
        >>> particles1.x = [3.0, 4.0, 5.0] | units.m
        >>> particles1.copy_values_of_attribute_to("x", particles2)
        >>> print particles2.x
        [3.0, 4.0] m

        """
        channel = self.new_channel_to(particles)
        channel.copy_attributes([attribute_name])

    def copy_values_of_attributes_to(self, attribute_names, particles):
        channel = self.new_channel_to(particles)
        channel.copy_attributes(attribute_names)

    def new_channel_to(self, other, attributes=None, target_names=None):
        return ParticleInformationChannel(self, other, attributes, target_names)

    def __add__(self, particles):
        """
        Returns a particle subset, composed of the given
        particle(s) and this particle set. Attribute values are
        not stored by the subset. The subset provides a view
        on two or more sets of particles.

        :parameter particles: (set of) particle(s) to be added to self.

        >>> particles = Particles(4)
        >>> particles1 = particles[:2]
        >>> particles1.x = [1.0, 2.0] | units.m
        >>> particles2 = particles[2:]
        >>> particles2.x = [3.0, 4.0] | units.m
        >>> new_set = particles1 + particles2
        >>> new_set  # doctest:+ELLIPSIS
        <amuse.datamodel.particles.ParticlesSubset object at 0x...>
        >>> print len(new_set)
        4
        >>> print new_set.x
        [1.0, 2.0, 3.0, 4.0] m
        """
        if isinstance(particles, Particle):
            particles = particles.as_set()
        original_particles_set = self._original_set()
        if not original_particles_set is particles._original_set():
            raise exceptions.AmuseException("Can't create new subset from particles belonging to "
                "separate particle sets. Try creating a superset instead.")
        keys = list(self.key) + list(particles.key)
        new_set = ParticlesSubset(original_particles_set, keys)
        if new_set.has_duplicates():
            raise exceptions.AmuseException("Unable to add a particle, because it was already part of this set.")
        return new_set

    def __or__(self, particles):
        """
        Returns a particle superset, composed of the given
        particle(s) and this particle set.

        :parameter particles: (set of) particle(s) to be added to self.

        >>> particles1 = Particles(2)
        >>> particles1.x = [1.0, 2.0] | units.m
        >>> particles2 = Particles(2)
        >>> particles2.x = [3.0, 4.0] | units.m
        >>> new_set = particles1 | particles2
        >>> new_set  # doctest:+ELLIPSIS
        <amuse.datamodel.particles.ParticlesSuperset object at 0x...>
        >>> print len(new_set)
        4
        >>> print new_set.x
        [1.0, 2.0, 3.0, 4.0] m
        """
        if isinstance(particles, Particle):
            particles = particles.as_set()

        original_particles_set1 = self._original_set()
        original_particles_set2 = particles._original_set()

        return ParticlesSuperset((original_particles_set1, original_particles_set2))

    def __sub__(self, particles):
        """
        Returns a subset of the set without the given particle(s)
        Attribute values are not stored by the subset. The subset
        provides a view on two or more sets of particles.

        :parameter particles: (set of) particle(s) to be subtracted from self.

        >>> particles = Particles(4)
        >>> particles.x = [1.0, 2.0, 3.0, 4.0] | units.m
        >>> junk = particles[2:]
        >>> new_set = particles - junk
        >>> new_set  # doctest:+ELLIPSIS
        <amuse.datamodel.particles.ParticlesSubset object at 0x...>
        >>> print len(new_set)
        2
        >>> print new_set.x
        [1.0, 2.0] m
        >>> print particles.x
        [1.0, 2.0, 3.0, 4.0] m
        """
        if isinstance(particles, Particle):
            particles = particles.as_set()
        new_keys = []
        new_keys.extend(self.get_all_keys_in_store())
        subtract_keys = particles.get_all_keys_in_store()
        for key in subtract_keys:
            if key in new_keys:
                new_keys.remove(key)
            else:
                raise exceptions.AmuseException("Unable to subtract a particle, because it is not part of this set.")
        return self._subset(new_keys)

    def add_particles(self, particles):
        """
        Adds particles from the supplied set to this set. Attributes
        and values are copied over.

        .. note::
            For performance reasons the particles
            are not checked for duplicates. When the same particle
            is part of both sets errors may occur.

        :parameter particles: set of particles to copy values from

        >>> particles1 = Particles(2)
        >>> particles1.x = [1.0, 2.0] | units.m
        >>> particles2 = Particles(2)
        >>> particles2.x = [3.0, 4.0] | units.m
        >>> particles1.add_particles(particles2)  # doctest:+ELLIPSIS
        <amuse.datamodel.particles.ParticlesSubset object at 0x...>
        >>> print len(particles1)
        4
        >>> print particles1.x
        [1.0, 2.0, 3.0, 4.0] m
        """
        if len(particles) == 0:
            return ParticlesSubset(self._original_set(),[])

        attributes = particles.get_attribute_names_defined_in_store()
        attributes= [x for x in attributes if x not in self._derived_attributes]
        
        indices = particles.get_all_indices_in_store()
        keys =  particles.get_all_keys_in_store()
        values = particles.get_values_in_store(indices, attributes)
        values = map(self._convert_from_entities_or_quantities, values)
        converted = []
        for x in values:
            if isinstance(x, LinkedArray):
                converted.append(x.copy_with_link_transfer(particles, self))
            else:
                converted.append(x)
        try:
            self.add_particles_to_store(keys, attributes, converted)
        except exceptions.MissingAttributesAmuseException as caught_exception:
            for attribute_name in caught_exception.missing_attributes:
                if attribute_name in particles._derived_attributes:
                    attributes.append(attribute_name)
                    converted.append(getattr(particles, attribute_name))
                else:
                    raise
            self.add_particles_to_store(keys, attributes, converted)
        return ParticlesSubset(self._original_set(),keys)



    def add_particle(self, particle):
        """
        Add one particle to the set.

        :parameter particle: particle to add

        >>> particles = Particles()
        >>> print len(particles)
        0
        >>> particle = Particle()
        >>> particle.x = 1.0 | units.m
        >>> particles.add_particle(particle)  # doctest:+ELLIPSIS
        <amuse.datamodel.particles.Particle object at ...>
        >>> print len(particles)
        1
        >>> print particles.x
        [1.0] m

        """
        return self.add_particles(particle.as_set())[0]

    def ensure_presence_of(self, x):
        """
        Add one particle to the set, but only if not already in set.

        :parameter particle: particle to add

        >>> particles = Particles()
        >>> print len(particles)
        0
        >>> particle = Particle()
        >>> p1=particles.ensure_presence_of(particle)  # doctest:+ELLIPSIS
        >>> print len(particles)
        1
        >>> p2=particles.ensure_presence_of(particle)  # doctest:+ELLIPSIS
        >>> print len(particles)
        1
        >>> p1==p2
        True
        >>> particle2 = Particle()
        >>> set=particle.as_set()
        >>> p3=set.add_particle(particle2)
        >>> set2=particles.ensure_presence_of(set)
        >>> len(particles)
        2
        >>> len(set2)
        2


        """
        try:
          not_in_set=x-x.get_intersecting_subset_in(self)
          self.add_particles(not_in_set)
          return x.get_intersecting_subset_in(self)
        except:  
          if x in self:
              return x.as_particle_in_set(self)
          else:
              return self.add_particle(x)

    def remove_particles(self, particles):
        """
        Removes particles from the supplied set from this set.

        :parameter particles: set of particles to remove from this set

        >>> particles1 = Particles(2)
        >>> particles1.x = [1.0, 2.0] | units.m
        >>> particles2 = Particles()
        >>> particles2.add_particle(particles1[0]) # doctest:+ELLIPSIS
        <amuse.datamodel.particles.Particle object at ...>
        >>> particles1.remove_particles(particles2)
        >>> print len(particles1)
        1
        >>> print particles1.x
        [2.0] m
        """
        
        if len(particles) == 0:
            return
        
        indices = self.get_indices_of_keys(particles.get_all_keys_in_store())
        self.remove_particles_from_store(indices)



    def remove_particle(self, particle):
        """
        Removes a particle from this set.

        Result is undefined if particle is not part of the set

        :parameter particle: particle to remove from this set

        >>> particles1 = Particles(2)
        >>> particles1.x = [1.0, 2.0] | units.m
        >>> particles1.remove_particle(particles1[0])
        >>> print len(particles1)
        1
        >>> print particles1.x
        [2.0] m
        """
        self.remove_particles(particle.as_set())

    def synchronize_to(self, other_particles):
        """
        Synchronize the particles of this set
        with the contents of the provided set.

        After this call the `other_particles` set will have
        the same particles as this set.

        This call will check if particles have been removed or
        added it will not copy values of existing particles
        over.

        :parameter other_particles: particle set wich has to be updated

        >>> particles = Particles(2)
        >>> particles.x = [1.0, 2.0] | units.m
        >>> copy = particles.copy()
        >>> new_particle = Particle()
        >>> new_particle.x = 3.0 | units.m
        >>> particles.add_particle(new_particle)# doctest:+ELLIPSIS
        <amuse.datamodel.particles.Particle object at ...>
        >>> print particles.x
        [1.0, 2.0, 3.0] m
        >>> print copy.x
        [1.0, 2.0] m
        >>> particles.synchronize_to(copy)
        >>> print copy.x
        [1.0, 2.0, 3.0] m

        """
        other_keys = set(other_particles.get_all_keys_in_store())
        my_keys = set(self.get_all_keys_in_store())
        added_keys = numpy.asarray(list(my_keys - other_keys))
        removed_keys = other_keys - my_keys
        removed_keys = list(removed_keys)
       
        if len(removed_keys) > 0:
            other_particles.remove_particles_from_store(other_particles.get_indices_of_keys(removed_keys))
            
        if len(added_keys) > 0:
            added_indices = self.get_indices_of_keys(added_keys)
            if len(added_indices) > 1:
                sort_indices = numpy.argsort(added_indices)
                added_indices = added_indices[sort_indices]
                added_keys = added_keys[sort_indices]
                
            attributes = self.get_attribute_names_defined_in_store()
            attributes= [x for x in attributes if x not in other_particles._derived_attributes]
            values = self.get_values_in_store(added_indices, attributes)
            converted = []
            memento = {}

            for x in values:
                if isinstance(x, LinkedArray):
                    converted.append(x.copy_with_link_transfer(self._original_set(), other_particles, True, memento))
                    #converted.append(x.copy(memento, False))
                else:
                    converted.append(x)
            try:
                other_particles.add_particles_to_store(added_keys, attributes, converted)
            except exceptions.MissingAttributesAmuseException as caught_exception:
                for attribute_name in caught_exception.missing_attributes:
                    if attribute_name in self._derived_attributes:
                        attributes.append(attribute_name)
                        converted.append(getattr(self._subset(added_keys), attribute_name))
                    else:
                        raise
                other_particles.add_particles_to_store(added_keys, attributes, converted)





    def compressed(self):
        return self

    def get_valid_particles_mask(self):
        return numpy.ones(len(self), dtype = numpy.bool)

    def as_set(self):
        """
        Returns a subset view on this set. The subset
        will contain all particles of this set.

        >>> particles = Particles(3)
        >>> particles.x = [1.0, 2.0, 3.0] | units.m
        >>> subset = particles.as_set()
        >>> print subset.x
        [1.0, 2.0, 3.0] m
        >>> print particles.x
        [1.0, 2.0, 3.0] m
        """
        return self._subset(self.get_all_keys_in_store())


    def select(self, selection_function, attributes):
        """
        Returns a subset view on this set. The subset
        will contain all particles for which the selection
        function returned True. The selection function
        is called with scalar quantities defined by
        the attributes parameter

        >>> particles = Particles(3)
        >>> particles.mass = [10.0, 20.0, 30.0] | units.kg
        >>> particles.x = [1.0, 2.0, 3.0] | units.m
        >>> subset = particles.select(lambda x : x > 15.0 | units.kg, ["mass"])
        >>> print subset.mass
        [20.0, 30.0] kg
        >>> print subset.x
        [2.0, 3.0] m

        """
        keys = self.get_all_keys_in_store()

        #values = self._get_values(keys, attributes) #fast but no vectors
        values = [getattr(self, x) for x in attributes]
        selected_keys = []
        for index in range(len(keys)):
            key = keys[index]
            arguments = [None] * len(attributes)
            for attr_index, attribute in enumerate(attributes):
                arguments[attr_index] = values[attr_index][index]
            if selection_function(*arguments):
                selected_keys.append(key)
        return self._subset(selected_keys)

    def select_array(self, selection_function, attributes = ()):
        """
        Returns a subset view on this set. The subset
        will contain all particles for which the selection
        function returned True. The selection function
        is called with a vector quantities containing all
        the values for the attributes parameter.

        This function can be faster than the select function
        as it works on entire arrays. The selection_function
        is called once.

        >>> particles = Particles(3)
        >>> particles.mass = [10.0, 20.0, 30.0] | units.kg
        >>> particles.x = [1.0, 2.0, 3.0] | units.m
        >>> subset = particles.select_array(lambda x : x > 15.0 | units.kg, ["mass"])
        >>> print subset.mass
        [20.0, 30.0] kg
        >>> print subset.x
        [2.0, 3.0] m


        >>> particles = Particles(1000)
        >>> particles.x = units.m.new_quantity(numpy.arange(1,1001))
        >>> subset = particles.select_array(lambda x : x > (500 | units.m), ("x",) )
        >>> print len(subset)
        500
        """
        keys = self.get_all_keys_in_store()
        #values = self._get_values(keys, attributes) #fast but no vectors
        quantities = [getattr(self, x) for x in attributes]
        selections = selection_function(*quantities)
        selected_keys =  numpy.compress(selections, keys)

        return self._subset(selected_keys)

    def difference(self, other):
        """
        Returns a new subset containing the difference between
        this set and the provided set.

        >>> particles = Particles(3)
        >>> particles.mass = [10.0, 20.0, 30.0] | units.kg
        >>> particles.x = [1.0, 2.0, 3.0] | units.m
        >>> subset = particles.select(lambda x : x > 15.0 | units.kg, ["mass"])
        >>> less_than_15kg = particles.difference(subset)
        >>> len(subset)
        2
        >>> len(less_than_15kg)
        1

        """
        return self.as_set().difference(other)

    def get_timestamp(self):
        return None

    def has_duplicates(self):
        """
        Returns True when a set contains a particle with the
        same key more than once. Particles with the same
        key are interpreted as the same particles.

        >>> particles = Particles()
        >>> p1 = particles.add_particle(Particle(1))
        >>> p2 = particles.add_particle(Particle(2))
        >>> particles.has_duplicates()
        False
        >>> p3 = particles.add_particle(Particle(1))
        >>> particles.has_duplicates()
        True
        >>> p3 == p1
        True
        """
        return len(self) != len(set(self.get_all_keys_in_store()))




    def _subset(self, keys):
        return ParticlesSubset(self._original_set(), keys)

    def _masked_subset(self, keys):
        return ParticlesMaskedSubset(self._original_set(), keys)

    def reversed(self):
        """
        Returns a subset with the same particles, but with reversed
        sequenctial order (the first particle will become last)

        >>> particles = Particles(3)
        >>> particles.radius = [1.0, 2.0, 3.0] | units.m
        >>> r = particles.reversed()
        >>> print r.radius
        [3.0, 2.0, 1.0] m

        """

        keys = self.get_all_keys_in_store()
        return self._subset(keys[::-1])

    def sorted_by_attribute(self, attribute, kind='mergesort'):
        """
        Returns a subset with the same particles, but sorted
        using the given attribute name

        :argument: kind, the sort method for supported kinds see
            the numpy.sort documentation


        >>> particles = Particles(3)
        >>> particles.mass = [2.0, 3.0, 1.0] | units.kg
        >>> particles.radius = [1.0, 2.0, 3.0] | units.m
        >>> sorted = particles.sorted_by_attribute('mass')
        >>> print sorted.mass
        [1.0, 2.0, 3.0] kg
        >>> print sorted.radius
        [3.0, 1.0, 2.0] m
        """
        return self.__getitem__(getattr(self, attribute).argsort(kind=kind))


    def sorted_by_attributes(self, *attributes):
        """
        Returns a subset with the same particles, but sorted
        using the given attribute names. The last attribute name
        in the call is used for the primary sort order, the
        second-to-last attribute name for the secondary sort order,
        and so on. See also numpy.lexsort


        >>> particles = Particles(4)
        >>> particles.mass = [2.0, 3.0, 1.0, 4.0] | units.kg
        >>> particles.radius = [3.0, 2.0, 1.0, 2.0] | units.m
        >>> sorted = particles.sorted_by_attributes('mass', 'radius')
        >>> print sorted.radius
        [1.0, 2.0, 2.0, 3.0] m
        >>> print sorted.mass
        [1.0, 3.0, 4.0, 2.0] kg
        """
        indices = self.get_all_indices_in_store()
        keys = self.get_all_keys_in_store()
        values = self.get_values_in_store(indices, attributes)
        values = [x.number for x in values]
        sorted_indices =  numpy.lexsort(values)

        return self._subset(keys[sorted_indices])

    def __dir__(self):
        """
        Utility function for introspection of paricle objects

        >>> particles = Particles(3)
        >>> particles.mass = [10.0, 20.0, 30.0] | units.kg
        >>> particles.x = [1.0, 2.0, 3.0] | units.m
        >>> print 'mass' in dir(particles)
        True
        >>> print 'x' in dir(particles)
        True

        """
        result = []
        result.extend(dir(type(self)))
        result.extend(self._attributes_for_dir())
        return result


    def __contains__(self, particle):
        """
        Check if a particle is part of a set

        >>> particles = Particles(3)
        >>> particles.mass = [10.0, 20.0, 30.0] | units.kg
        >>> p1 = particles[1]
        >>> print p1 in particles
        True
        >>> p4 = Particle()
        >>> print p4 in particles
        False

        """
        
        return isinstance(particle, Particle) and self.has_key_in_store(particle.key)


    def all_attributes(self):
        result = []
        result.append('key')
        result.extend(self._attributes_for_dir())
        return result


    def is_empty(self):
        return self.__len__()==0

    def get_intersecting_subset_in(self, other):
        selected_keys = [x for x in self.get_all_keys_in_store() if other.has_key_in_store(x)]
        return other._subset(selected_keys)

    def _as_masked_subset_in(self, other):
        keys = numpy.ma.array(self.get_all_keys_in_store(), dtype='uint64')
        keys.mask = ~self.get_valid_particles_mask()
        return other._masked_subset(keys)

    def random_sample(self, number_of_particles):
        return self.__getitem__(random.sample(range(len(self)), number_of_particles))


    def empty_particles_set_string(self, attributes = ()):
        result = 'empty particles set ('+self.__class__.__name__+')'
        if len(attributes) > 0:
            sorted_attributes = sorted(attributes)
            sorted_attributes.insert(0, 'key')
            header = 'attributes:'
            result += '\n' + header
            for x in self.divide_attributes(sorted_attributes):
                result += '\n'
                result += ' ' * len(header)
                result += ','.join(x)
        return result

    def divide_attributes(self, attributes = (), n = 3):
        y = []
        for x in attributes:
            y.append(x)
            if len(y) == n:
                yield y
                y = []
            
    def get_all_values_of_attribute_in_store(self, attribute):
        return self.get_values_in_store(None, [attribute])[0]
        

    def __setattr__(self, name_of_the_attribute, value):
        value = self.check_attribute(value)
        if name_of_the_attribute in self._derived_attributes:
            self._derived_attributes[name_of_the_attribute].set_values_for_entities(self, value)
        else:
            self.set_values_in_store(None, [name_of_the_attribute], [self._convert_from_entities_or_quantities(value)])

    def get_containing_set(self):
        return self


class Particles(AbstractParticleSet):
    """
    A set of particles. Attributes and values are stored in
    a private storage model. This storage model can store
    the values in the python memory space, in the memory space
    of the code or in a HDF5 file. By default the storage
    model is in memory.



    """
    def __init__(self, size = 0, storage = None, keys = None, keys_generator = None, particles = None, is_working_copy = True, **attributes):
        AbstractParticleSet.__init__(self)

        self._private.version = 0
        self._private.is_working_copy = is_working_copy

        if storage is None:
            self._private.attribute_storage = get_in_memory_attribute_storage_factory()()
        else:
            self._private.attribute_storage = storage


        if keys_generator is None:
            keys_generator = base.UniqueKeyGenerator

        if not particles is None:
            if isinstance(particles,AbstractParticleSet):
                self.add_particles(particles)
            else:
                for x in iter(particles):
                    self.add_particle(x)
        elif size > 0:
            if not keys is None:
                if len(keys) != size:
                    raise Exception('keys and size was specified in the creation of a particle set, but the length of the keys is not equal to the size')
                else:
                    particle_keys = keys
            else:
                particle_keys = keys_generator.next_set_of_keys(size)

            self.add_particles_to_store(particle_keys)
        elif not keys is None:
            self.add_particles_to_store(keys)
        elif len(attributes) > 0:
            number_of_attributes = 0
            for attributevalue in attributes.values():
                if is_quantity(attributevalue):
                    if attributevalue.is_scalar():
                        number_of_attributes = max(number_of_attributes, 1)
                    else:
                        number_of_attributes = max(number_of_attributes, len(attributevalue))
                else:
                    try:
                        if isinstance(attributevalue, str):
                            number_of_attributes = max(number_of_attributes,1)
                        else:
                            number_of_attributes = max(number_of_attributes, len(attributevalue))
                    except: #fails for numbers
                        number_of_attributes = max(number_of_attributes,1)

            particle_keys = keys_generator.next_set_of_keys(number_of_attributes)
            self.add_particles_to_store(particle_keys)


        self._private.version = 0

        if len(attributes) > 0:
            attributenames = []
            attributevalues = []
            for attributename, attributevalue in attributes.items():
                attributenames.append(attributename)
                attributevalues.append(attributevalue)

            self.set_values_in_store(self.get_all_indices_in_store(), attributenames, attributevalues)

        self._private.previous = None
        self.collection_attributes.timestamp = None


    def __getitem__(self, index):
        if index is None:
            keys = self.get_all_keys_in_store()
            index = self.get_all_indices_in_store()
        else:
            keys = self.get_all_keys_in_store()[index]
            index = self.get_all_indices_in_store()[index]

        if hasattr(keys, '__iter__'):
            return self._subset(keys)
        else:
            return Particle(keys, self, index, self._get_version())


    def _get_version(self):
        return self._private.version

    def __iter__(self):
        keys =  self.get_all_keys_in_store()
        indices = self.get_all_indices_in_store()
        version = self._get_version()

        for i in range(len(keys)):
            yield Particle(keys[i], self,  indices[i], version)

    def savepoint(self, timestamp=None, format = 'memory', **attributes):
        if format == 'memory':
            try:
                instance = type(self)(is_working_copy = False)
                instance._private.attribute_storage = self._private.attribute_storage.copy()
            except:
                instance = self.copy()
                instance._private.is_working_copy = False
                
            instance.collection_attributes.timestamp = timestamp

            for name, value in attributes.items():
                setattr(instance.collection_attributes, name, value)
        else:
            raise Exception("{0} not supported, only 'memory' savepoint supported".format(format))
        instance._private.previous = self._private.previous
        instance._private.version = 0
        self._private.previous = instance
        return instance

    def new_working_copy(self):
        if self._private.is_working_copy:
            previous = self._private.previous
            if previous is None:
                raise Exception("you have not savepoint for this set, you cannot create a working copy please use copy instead".format(format))
        else:
            previous = self
        result = previous.copy()
        result._private.previous = previous
        return result

    def get_timestamp(self):
        return self.collection_attributes.timestamp

    def iter_history(self):
        if self._private.is_working_copy:
            current = self._private.previous
        else:
            current = self

        while not current is None:
            yield current
            current = current._private.previous


    def get_state_at_timestamp(self, timestamp):
        previous_timestamp = None
        states_and_distances = []
        for state in self.iter_history():
            timestamp_of_state = state.get_timestamp()
            if timestamp_of_state is None:
                continue
            distance = abs(timestamp_of_state - timestamp)
            states_and_distances.append((state, distance,))

        if len(states_and_distances) == 0:
            raise exceptions.AmuseException("You asked for a state at timestamp '{0}', but the set does not have any saved states so this state cannot be returned")

        accompanying_state, min_distance = states_and_distances[0]
        for state, distance  in states_and_distances:
            if distance < min_distance:
                min_distance = distance
                accompanying_state = state

        return accompanying_state

    def previous_state(self):
        return self._private.previous

    @property
    def history(self):
        return reversed(list(self.iter_history()))

    def get_timeline_of_attribute(self, particle_key, attribute):
        timeline = []
        for x in self.history:
            if x.has_key_in_store(particle_key):
                index = x.get_indices_of_keys([particle_key])[0]
                timeline.append((x.collection_attributes.timestamp, x._get_value_of_attribute(x[index], index, attribute)))
        return timeline

    def get_timeline_of_attribute_as_vector(self, particle_key, attribute):
        timeline = AdaptingVectorQuantity()
        chrono_values = AdaptingVectorQuantity()
        for x in self.history:
            if x.has_key_in_store(particle_key):
                index = x.get_indices_of_keys([particle_key])[0]
                timeline.append(x.collection_attributes.timestamp)
                chrono_values.append(x._get_value_of_attribute(x[index], index, attribute))
        return timeline, chrono_values

    def get_timeline_of_attributes(self, particle_key, attributes):
        result = [[] for x in range(len(attributes)+1)]
        units = [None for x in range(len(attributes)+1)]

        for x in self.history:
            if x.has_key_in_store(particle_key):
                index = x.get_indices_of_keys([particle_key])[0]
                if  units[0] is None:
                    units[0] = x.collection_attributes.timestamp.unit
                for i, attribute in enumerate(attributes):
                    quantity = x._get_value_of_attribute(x[index], index, attribute)
                    if  units[i+1] is None:
                        units[i+1] = quantity.unit
                    result[i+1].append(quantity.value_in(units[i+1]))

        return list(map(lambda value,unit : unit.new_quantity(value), result, units))


   

    def remove_attribute_from_store(self, name):
        self._private.attribute_storage.remove_attribute_from_store(name)
        
    def add_particles_to_store(self, keys, attributes = [], values = []):
        self._private.attribute_storage.add_particles_to_store(keys, attributes, values)
        self._private.version += 1


    def remove_particles_from_store(self, indices):
        self._private.attribute_storage.remove_particles_from_store(indices)
        self._private.version += 1

    def get_values_in_store(self, indices, attributes):
        missing_attributes = set(attributes) - set(self.get_attribute_names_defined_in_store()) - set(["index_in_code"])

        if len(missing_attributes) == 0:
            return self._private.attribute_storage.get_values_in_store(indices, attributes)

        defined_attributes = list(set(attributes) - missing_attributes)
        defined_values = dict(list(zip(
            defined_attributes,
            self._private.attribute_storage.get_values_in_store(indices, defined_attributes)
        )))
        #print missing_attributes, "shape" in missing_attributes
        #if "shape" in missing_attributes:
        #    import traceback
        #    traceback.print_stack()
        #    raise Exception("hello is this ok????")
        subset = self[indices]
        tmp = [defined_values[attribute] if attribute in defined_values else subset._get_derived_attribute_value(attribute) for attribute in attributes]
        return tmp

    def get_values_in_store_async(self, indices, attributes):
        return self._private.attribute_storage.get_values_in_store_async(indices, attributes)


    def get_indices_of_keys(self, keys):
        return self._private.attribute_storage.get_indices_of(keys)

    def set_values_in_store(self, indices, attributes, values):
        self._private.attribute_storage.set_values_in_store(indices, attributes, values)

    def set_values_in_store_async(self, indices, attributes, values):
        return self._private.attribute_storage.set_values_in_store_async(indices, attributes, values)

    def get_attribute_names_defined_in_store(self):
        return self._private.attribute_storage.get_defined_attribute_names()

    def get_settable_attribute_names_defined_in_store(self):
        return self._private.attribute_storage.get_defined_settable_attribute_names()

    def get_all_keys_in_store(self):
        return self._private.attribute_storage.get_all_keys_in_store()

    def get_all_indices_in_store(self):
        return self._private.attribute_storage.get_all_indices_in_store()

    def has_key_in_store(self, key):
        return self._private.attribute_storage.has_key_in_store(key)

    def get_value_in_store(self, index, attribute):
        return self._private.attribute_storage.get_value_in_store(index, attribute)

    def can_extend_attributes(self):
        return self._private.attribute_storage.can_extend_attributes()


    def _remove_indices_in_attribute_storage(self, indices):
        self._private.attribute_storage._remove_indices(indices)
        self._private.version += 1

    def _add_indices_in_attribute_storage(self, indices):
        self._private.attribute_storage._add_indices(indices)
        self._private.version += 1

    @staticmethod
    def is_quantity():
        return False
        
    def new_particle(self, key = None, **keyword_arguments):
        return self.add_particle(Particle(key = key, **keyword_arguments))
        

class BoundSupersetParticlesFunctionAttribute(object):
    def  __init__(self, name, superset):
        self.name = name
        self.superset = superset
        self.subsetfunctions = []

    def add_subsetfunction(self, callable):
        self.subsetfunctions.append(callable)

    def __call__(self, *list_arguments, **keyword_arguments):
        subset_results = []
        for x in self.subsetfunctions:
            subset_results.append(x(*list_arguments, **keyword_arguments))

        if subset_results[0] is None:
            return None
        if isinstance(subset_results[0], AbstractParticleSet):
            return ParticlesSuperset(subset_results)
        if hasattr(subset_results[0], 'unit'):
            result = AdaptingVectorQuantity()
            for one_result in subset_results:
                result.extend(one_result)
            return result
        return [item for one_result in subset_results for item in one_result]

class DerivedSupersetAttribute(DerivedAttribute):


    def __init__(self, name):
        self.name = name

    def get_values_for_entities(self, superset):
        result = None
        offset = 0
        for subset in superset._private.particle_sets:

            subset_result =  getattr(subset, self.name)
            if hasattr(subset_result, '__call__'):
                if len(subset) > 0:
                    if result is None:
                        result = BoundSupersetParticlesFunctionAttribute(self.name, superset)
                    result.add_subsetfunction(subset_result)

            elif hasattr(subset_result, 'unit'):
                if len(subset_result) == 0:
                    continue
                if result is None:
                    shape = [len(superset),] + list(subset_result.shape[1:])
                    result = VectorQuantity.zeros(shape, subset_result.unit)
                    offset = 0
                try:
                    result[offset:len(subset_result)+offset] = subset_result
                except ValueError:
                    raise AttributeError("Subsets return incompatible quantities for attribute '{0}', attribute cannot be queried from the superset".format(self.name))
                offset += len(subset_result)

            elif hasattr(subset_result, 'dtype'):
                if len(subset_result) == 0:
                    continue
                if result is None:
                    shape = [len(superset),] + list(subset_result.shape[1:])
                    result = numpy.zeros(shape, dtype=subset_result.dtype)
                    offset = 0
                try:
                    result[offset:len(subset_result)+offset] = subset_result
                except ValueError:
                    raise AttributeError("Subsets return incompatible quantities for attribute '{0}', attribute cannot be queried from the superset".format(self.name))
                offset += len(subset_result)
            else:
                raise exceptions.AmuseException("cannot handle this type of attribute on supersets yet")
        return result

    def set_values_for_entities(self, superset, value):
        raise exceptions.AmuseException("cannot set value of attribute '{0}'")

    def get_value_for_entity(self, superset, particle, index):
        raise exceptions.AmuseException("Internal AMUSE error, a single entity (Particle) should always be bound to the subset and not the superset")

    def set_value_for_entity(self, superset, key, value):
        raise exceptions.AmuseException("Internal AMUSE error, a single entity (Particle) should always be bound to the subset and not the superset")

class ParticlesSuperset(AbstractParticleSet):
    """A superset of particles. Attribute values are not
    stored by the superset. The superset provides a view
    on two or more sets of particles.

    Superset objects are not supposed to be created
    directly. Instead use the ``union`` methods.

    >>> p1 = Particles(3)
    >>> p1.mass = [10.0, 20.0, 30.0] | units.kg
    >>> p2 = Particles(3)
    >>> p2.mass = [40.0, 50.0, 60.0] | units.kg
    >>> p = ParticlesSuperset([p1, p2])
    >>> print len(p)
    6
    >>> print p.mass
    [10.0, 20.0, 30.0, 40.0, 50.0, 60.0] kg
    >>> p[4].mass = 70 | units.kg
    >>> print p.mass
    [10.0, 20.0, 30.0, 40.0, 70.0, 60.0] kg
    >>> p2[1].mass
    quantity<70.0 kg>
    >>> cp = p.copy()
    >>> print len(cp)
    6
    >>> print cp.mass
    [10.0, 20.0, 30.0, 40.0, 70.0, 60.0] kg
    """

    def __init__(self, particle_sets, index_to_default_set=None, names = None):
        AbstractParticleSet.__init__(self)

        if not names is None:
            self._private.mapping_from_name_to_set = {}
            for name, particle_set in zip(names, particle_sets):
                self._private.mapping_from_name_to_set[name] = particle_set

        self._private.particle_sets = list(particle_sets)
        self._private.index_to_default_set = index_to_default_set


        names_of_derived_attributes_in_all_subsets = None
        for subset in particle_sets:
            derived_attribute_names = set(subset._derived_attributes.keys())
            if names_of_derived_attributes_in_all_subsets is None:
                names_of_derived_attributes_in_all_subsets = set(derived_attribute_names)
            else:
                names_of_derived_attributes_in_all_subsets &= derived_attribute_names

        names_of_derived_attributes_in_all_subsets -= set(self.GLOBAL_DERIVED_ATTRIBUTES.keys())
        for name in names_of_derived_attributes_in_all_subsets:
            self._derived_attributes[name] = DerivedSupersetAttribute(name)

        self._private.version = -1
        self._ensure_updated_set_properties()

        if self.has_duplicates():
            raise exceptions.AmuseException("Unable to add a particle, because it was already part of this set.")


    def _ensure_updated_set_properties(self):
        if self._private.version == self._get_subsets_version():
            return

        self._private.version = self._get_subsets_version()
        self._private.length = numpy.sum([len(x) for x in  self._private.particle_sets])
        self._private.indices = numpy.arange(self._private.length)
        self._private.keys = self._get_concatenated_keys_in_store()
        self._private.key_to_index = {}

        d = self._private.key_to_index
        index = 0
        for x in self._private.keys:
            d[x] = index
            index += 1

    def can_extend_attributes(self):
        for x in self._private.particle_sets:
            if not x.can_extend_attributes():
                return False
        return True

    def __len__(self):
        self._ensure_updated_set_properties()

        return self._private.length

    def __iter__(self):
        for set in self._private.particle_sets:
            for particle in set:
                yield particle

    def _get_subsets_version(self):
        versions = [[x._get_version()] for x in self._private.particle_sets]
        return numpy.sum(versions)

    def _get_version(self):
        self._ensure_updated_set_properties()

        return self._private.version


    def __getitem__(self, index):
        self._ensure_updated_set_properties()

        offset = 0

        if isinstance(index, str):
            return self.get_subset(index)
        else:
            keys = self.get_all_keys_in_store()[index]
            if hasattr(keys, '__iter__'):
                return self._subset(keys)
            else:
                index = self.get_all_indices_in_store()[index]
                for set in self._private.particle_sets:
                    length = len(set)
                    if index < (offset+length):
                        return set[index - offset]
                    offset += length
                raise Exception('index not found on superset')

    def _get_particle(self, key):
        if self.has_key_in_store(key):
            return self._get_subset_for_key(key)._get_particle(key)
        else:
            return None

    def _get_particle_unsave(self, key, index = -1):
        if index >= 0:
            offset, subset = self._get_offset_and_subset_for_index(index)
            index -= offset
            return subset._get_particle_unsave(key, subset.get_all_indices_in_store()[index])
        else:
            return self._get_subset_for_key(key)._get_particle_unsave(key)

    def _split_keys_over_sets(self, keys):
        split_sets = [ [] for x in self._private.particle_sets ]
        split_indices = [ [] for x in self._private.particle_sets ]


        if keys is None:
            offset = 0

            for setindex, x in enumerate(self._private.particle_sets):
                split_sets[setindex].extend(x.get_all_keys_in_store())
                split_indices[setindex].extend(range(offset, offset + len(x)))
                offset = offset + len(x)

        else:
            if isinstance(keys, set):
                keys_array = numpy.array(list(keys))
            else:
                keys_array = numpy.array(keys)

            indices_array = numpy.arange(len(keys_array))
            for setindex, x in enumerate(self._private.particle_sets):
                mask = self._in1d(keys_array, x.get_all_keys_in_store(), True)
                split_sets[setindex] =  keys_array[mask]
                split_indices[setindex] = indices_array[mask]

        return split_sets, split_indices


    def _split_indices_over_sets(self, indices):
        self._ensure_updated_set_properties()

        split_sets = [ [] for x in self._private.particle_sets ]
        split_indices = [ [] for x in self._private.particle_sets ]

        offset = 0
        if isinstance(indices, set):
            indices = numpy.array(list(indices))

        if indices is None or isinstance(indices,EllipsisType):
            offset = 0

            for setindex, x in enumerate(self._private.particle_sets):
                split_sets[setindex] = x.get_all_indices_in_store()
                split_indices[setindex] = numpy.arange(offset, offset + len(x))
                offset = offset + len(x)
        elif len(indices) == 0:
            for setindex, x in enumerate(self._private.particle_sets):
                split_sets[setindex] = []
                split_indices[setindex] = []
        else:
            result_indices_array = numpy.arange(len(indices))
            for setindex, x in enumerate(self._private.particle_sets):
                mask = numpy.logical_and( (indices >= offset) , (indices < (offset + len(x))) )
                indices_in_store = numpy.asarray(x.get_all_indices_in_store())
                split_sets[setindex] = indices_in_store[(indices-offset)[mask]]
                split_indices[setindex] = result_indices_array[mask]
                offset = offset + len(x)
        return split_sets, split_indices



    def add_particles_to_store(self, keys, attributes = [], values = []):
        if not self._private.index_to_default_set is None:
            self._private.particle_sets[self._private.index_to_default_set].add_particles_to_store(keys,
                attributes, values)
        else:
            raise exceptions.AmuseException("Cannot add particles to a superset")

    def remove_particles_from_store(self, indices):
        split_indices_in_subset, split_indices_in_input = self._split_indices_over_sets(indices)

        for indices_in_subset, set in zip(split_indices_in_subset, self._private.particle_sets):
            if len(indices_in_subset) == 0:
                continue

            set.remove_particles_from_store(indices_in_subset)


    def get_values_in_store(self, indices, attributes):
        split_indices_in_subset, split_indices_in_input = self._split_indices_over_sets(indices)

        indices_and_values = []

        for indices_in_subset, indices_in_input, set in zip(split_indices_in_subset, split_indices_in_input, self._private.particle_sets):
            if len(indices_in_subset) > 0:
                values_for_set = set.get_values_in_store(indices_in_subset, attributes)
                indices_and_values.append( (indices_in_input, values_for_set) )

        if indices is None or isinstance(indices, EllipsisType):
            resultlength = len(self)
        else:
            resultlength = len(indices)

        values = [[]] * len(attributes)
        units = [None] * len(attributes)
        converts = [lambda x : x] * len(attributes)
        for indices, values_for_set in indices_and_values:
            for valueindex, quantity in enumerate(values_for_set):
                resultvalue = values[valueindex]
                if len(resultvalue) == 0:
                    if not is_quantity(quantity):
                        dtype = quantity.dtype
                        converts[valueindex] = lambda x : x
                        units[valueindex] = None
                    else:
                        dtype = quantity.number.dtype
                        converts[valueindex] = quantity.unit.new_quantity
                        units[valueindex] = quantity.unit
                    shape = list(quantity.shape)
                    shape[0] = resultlength
                    resultvalue = numpy.zeros(shape,dtype=dtype)
                    values[valueindex] = resultvalue

                resultunit = units[valueindex]
                if not resultunit is None:
                    resultvalue[indices] = quantity.value_in(resultunit)
                else:
                    current_dtype = quantity.dtype
                    result_dtype = resultvalue.dtype
                    if (result_dtype.kind == 'S' or result_dtype.kind == 'U') and (current_dtype.itemsize > result_dtype.itemsize):
                        resultvalue = numpy.asarray(resultvalue, dtype = current_dtype)
                        values[valueindex] = resultvalue
                    resultvalue[indices] = quantity

        return list(map(lambda u,v : u(v), converts, values))

    def set_values_in_store(self, indices, attributes, values):
        split_indices_in_subset, split_indices_in_input = self._split_indices_over_sets(indices)
        if indices is None or indices is Ellipsis:
            len_indices = 0
            for x in split_indices_in_subset:
                len_indices += len(split_indices_in_subset)
        else:
            len_indices = len(indices)
        for indices_in_subset, indices_in_input, set in zip(split_indices_in_subset, split_indices_in_input, self._private.particle_sets):
            quantities = [None] * len(attributes)
            for valueindex, quantity in enumerate(values):
                if is_quantity(quantity):
                    if quantity.is_scalar():
                        numbers = [quantity.number]*len(indices_in_input)
                    elif quantity.is_vector() and len(quantity) < len_indices:
                        numbers = numpy.take([quantity.number]*len_indices,indices_in_input)
                    else:
                        numbers = numpy.take(quantity.number, indices_in_input)
                    quantities[valueindex] = quantity.unit.new_quantity(numbers)
                else:
                    if not hasattr(quantity, 'ndim'):
                        numbers = numpy.asarray([quantity]*len(indices_in_input))
                    elif len(quantity) < len_indices:
                        numbers = numpy.take([quantity]*len_indices,indices_in_input)
                    else:
                        numbers = numpy.take(quantity, indices_in_input)
                    quantities[valueindex] = numbers

            set.set_values_in_store(indices_in_subset, attributes, quantities)

    def get_attribute_names_defined_in_store(self):
        self._ensure_updated_set_properties()
        result = None
        for particle_set in self._private.particle_sets:
            if len(particle_set) > 0:
                attribute_names = set(particle_set.get_attribute_names_defined_in_store())
                if result is None:
                    result = attribute_names
                else:
                    result &= attribute_names
        if result is None:
            return []
        else:
            return list(result)

    def get_settable_attribute_names_defined_in_store(self):
        self._ensure_updated_set_properties()
        result = None
        for particle_set in self._private.particle_sets:
            if len(particle_set) > 0:
                attribute_names = set(particle_set.get_settable_attribute_names_defined_in_store())
                if result is None:
                    result = attribute_names
                else:
                    result &= attribute_names
        if result is None:
            return []
        else:
            return list(result)

    def get_all_keys_in_store(self):
        self._ensure_updated_set_properties()

        return self._private.keys

    def get_all_indices_in_store(self):
        self._ensure_updated_set_properties()

        return self._private.indices

    def get_indices_of_keys(self, keys):
        self._ensure_updated_set_properties()
        
        mapping_from_particle_to_index = self._private.key_to_index
        result = []
        notfoundkeys = []
        foundkeys = []
        for x in keys:
            try:
                result.append(mapping_from_particle_to_index[x])
                foundkeys.append(x)
            except KeyError:
                notfoundkeys.append(x)
        
        if not len(notfoundkeys) == 0:
            raise exceptions.KeysNotInStorageException(
                numpy.asarray(foundkeys), 
                numpy.asarray(result), 
                numpy.asarray(notfoundkeys)
            )
        return numpy.array(result)


    def has_key_in_store(self, key):
        for set in self._private.particle_sets:
            if set.has_key_in_store(key):
                return True
        return False


    def _original_set(self):
        return self

    def _get_subset_for_key(self, key):
        for set in self._private.particle_sets:
            if set.has_key_in_store(key):
                return set
        return None

    def _get_offset_and_subset_for_index(self, index):
        offset = 0
        for set in self._private.particle_sets:
            setlen = len(set)
            if index >= offset and index < (offset+setlen):
                return offset, set
            offset += setlen
        return None, None

    def _is_superset(self):
        return True

    def _in1d(self, ar1, ar2, assume_unique=False):
        """
        copied from numpy.in1d (nump 1.4), to be compatible with numpy 1.3.0
        """
        if not assume_unique:
            ar1, rev_idx = numpy.unique(ar1, return_inverse=True)
            ar2 = numpy.unique(ar2)

        ar = numpy.concatenate( (ar1, ar2) )
        order = ar.argsort(kind='mergesort')
        sar = ar[order]
        equal_adj = (sar[1:] == sar[:-1])
        flag = numpy.concatenate( (equal_adj, [False] ) )
        index = order.argsort(kind='mergesort')[:len( ar1 )]

        if assume_unique:
            return flag[index]
        else:
            return flag[index][rev_idx]

    def _get_concatenated_keys_in_store(self):
        result = []
        dtype = None
        for set in self._private.particle_sets:
            subsetkeys = set.get_all_keys_in_store()
            if dtype is None and len(set) > 0:
                dtype = subsetkeys.dtype
            result.extend(subsetkeys)

        return numpy.array(result, dtype = dtype)

    def get_subsets(self):
        return list(self._private.particle_sets)

    def get_subset(self, name):
        return self._private.mapping_from_name_to_set[name]

class ParticlesSubset(AbstractParticleSet):
    """A subset of particles. Attribute values are not
    stored by the subset. The subset provides a limited view
    to the particles.

    Particle subset objects are not supposed to be created
    directly. Instead use the ``to_set`` or ``select`` methods.

    """

    def __init__(self, particles, keys):
        AbstractParticleSet.__init__(self, particles)
        self._private.particles = particles
        self._private.keys = numpy.array(keys, dtype='uint64')
        self._private.set_of_keys = set(keys)

        self._private.version = -1
        self._private.indices = None


    def __getitem__(self, index):
        keys = self.get_all_keys_in_store()[index]
        if hasattr(keys, '__iter__'):
            return self._subset(keys)
        else:
            key = keys
            if key > 0 and key < 18446744073709551615: #2**64 - 1
                return self._private.particles._get_particle_unsave(key, self.get_all_indices_in_store()[index])
            else:
                return None

    def _get_version(self):
        return self._private.particles._get_version()

    def compressed(self):
        keys = self._private.keys
        return self._subset(keys[numpy.logical_and(keys > 0 ,  keys < 18446744073709551615)])

    def get_valid_particles_mask(self):
        keys = self._private.keys
        return numpy.logical_and(keys > 0 ,  keys < 18446744073709551615)

    def unconverted_set(self):
        return ParticlesSubset(self._private.particles.unconverted_set(), self._private.keys)

    def add_particles_to_store(self, keys, attributes = [], values = []):
        """
        Adds particles from to the subset, also
        adds the particles to the superset
        """
        self._private.keys = numpy.concatenate((self._private.keys,  numpy.array(keys,dtype='uint64')))
        self._private.set_of_keys |= set(keys)
        self._private.particles.add_particles_to_store(keys, attributes, values)

    def remove_particles_from_store(self, indices):
        """
        Removes particles from the subset, and removes particles
        from the super set
        """

        indices_to_remove = set(indices)

        index = 0
        index_in_local_list = []
        for x in self._private.indices:
            if x in indices_to_remove:
                index_in_local_list.append(index)
            index += 1
        set_to_remove = set(self._private.keys[index_in_local_list])
        self._private.keys = numpy.delete(self._private.keys,index_in_local_list)
        self._private.set_of_keys -= set_to_remove
        self._private.particles.remove_particles_from_store(indices)

        self._private.version = -1
        self._private.indices = None


    def get_values_in_store(self, indices, attributes):
        if indices is None or indices is Ellipsis:
            indices = self.get_all_indices_in_store()

        return self._private.particles.get_values_in_store(indices, attributes)

    def get_values_in_store_async(self, indices, attributes):
        if indices is None or indices is Ellipsis:
            indices = self.get_all_indices_in_store()

        return self._private.particles.get_values_in_store_async(indices, attributes)

    def set_values_in_store(self, indices, attributes, values):
        if indices is None or indices is Ellipsis:
            indices = self.get_all_indices_in_store()

        self._private.particles.set_values_in_store(indices, attributes, values)

    def set_values_in_store_async(self, indices, attributes, values):
        if indices is None or indices is Ellipsis:
            indices = self.get_all_indices_in_store()

        return self._private.particles.set_values_in_store_async(indices, attributes, values)

    def get_attribute_names_defined_in_store(self):
        return self._private.particles.get_attribute_names_defined_in_store()

    def get_settable_attribute_names_defined_in_store(self):
        return self._private.particles.get_settable_attribute_names_defined_in_store()

    def get_all_keys_in_store(self):
        self._sync_with_set()
            
        return self._private.keys

    def get_all_indices_in_store(self):
        self._sync_with_set()

        return self._private.indices

    def has_key_in_store(self, key):
        return key in self._private.set_of_keys

    def _original_set(self):
        return self._private.particles

    def get_timestamp(self):
        return self._original_set().get_timestamp()

    def get_indices_of_keys(self, keys):
        self._sync_with_set()
        
        return self._private.particles.get_indices_of_keys(keys)
        
    def _sync_with_set(self):
        if self._private.particles is None:
            return
        if not self._private.version == self._private.particles._get_version():
            try:
                self._private.indices = self._private.particles.get_indices_of_keys(self._private.keys)
                self._private.version = self._private.particles._get_version()
            except exceptions.KeysNotInStorageException as ex:
                self._private.indices = ex.found_indices
                self._private.keys = numpy.array(ex.found_keys, dtype='uint64')
                self._private.set_of_keys = set(self._private.keys)
                self._private.version = self._private.particles._get_version()

    def previous_state(self):
        return ParticlesSubset(self._private.particles.previous_state(), self._private.keys)

    def difference(self, other):
        self._sync_with_set()
        
        new_set_of_keys = self._private.set_of_keys.difference(other.as_set()._private.set_of_keys)
        return ParticlesSubset(self._private.particles, list(new_set_of_keys))

    def union(self, other):
        """
        Returns a new subset containing the union between
        this set and the provided set.

        >>> particles = Particles(3)
        >>> particles.mass = [10.0, 20.0, 30.0] | units.kg
        >>> subset1 = particles.select(lambda x : x > 25.0 | units.kg, ["mass"])
        >>> subset2 = particles.select(lambda x : x < 15.0 | units.kg, ["mass"])
        >>> union = subset1.union(subset2)
        >>> len(union)
        2
        >>> sorted(union.mass.value_in(units.kg))
        [10.0, 30.0]
        """

        self._sync_with_set()
        new_set_of_keys = self._private.set_of_keys.union(other.as_set()._private.set_of_keys)
        return ParticlesSubset(self._private.particles, list(new_set_of_keys))

    def as_set(self):
        return self

    def copy(self, memento = None, keep_structure = False, filter_attributes = lambda particle_set, x : True):
        self._sync_with_set()
        if keep_structure:
            result = ParticlesSubset(None, [])
            if memento is None:
                memento = dict()
            memento[id(self)] = result
            if id(self._private.particles) in memento:
                result._private.particles = memento[id(self._private.particles)]
            else:
                result._private.particles = self._private.particles.copy(memento, keep_structure, filter_attributes)
            result._private.keys = numpy.array(self._private.keys, dtype='uint64')
            result._private.set_of_keys = set(self._private.keys)

            result._private.collection_attributes = self._private.collection_attributes._copy_for_collection(result)

            return result
        else:
            return super(ParticlesSubset, self).copy(memento, keep_structure, filter_attributes)



class ParticlesMaskedSubset(ParticlesSubset):
    """A subset of particles. Attribute values are not
    stored by the subset. The subset provides a limited view
    to the particles. Unlike the normal subset the masked subset
    can store None values.
    """

    def __init__(self, particles, keys):
        AbstractParticleSet.__init__(self, particles)
        self._private.particles = particles

        self._private.keys = numpy.ma.array(keys, dtype='uint64').copy()
        validkeys = self._private.keys.compressed()
        if len(validkeys) > 0:
            self._private.set_of_keys = set(validkeys)
        else:
            self._private.set_of_keys = set([])

        self._private.version = -1
        self._private.indices = None

    def compressed(self):
        return self._subset(self._private.keys.compressed())

    def get_valid_particles_mask(self):
        return ~self._private.keys.mask

    def __iter__(self):
        original_set = self._original_set()
        for key in self._private.keys :
            if key is ma.masked:
                yield None
            else:
                yield original_set._get_particle_unsave(key)

    def __getitem__(self, index):
        keys = self.get_all_keys_in_store()[index]

        if not keys is ma.masked and hasattr(keys, '__iter__'):
            if numpy.all(~ keys.mask):
                return self._subset(keys.data)
            else:
                return ParticlesMaskedSubset(self._original_set(),keys)
        else:
            key = keys
            if not key is ma.masked:
                return self._original_set()._get_particle_unsave(key, self.get_all_indices_in_store()[index])
            else:
                return None

    def _get_version(self):
        return self._private.particles._get_version()

    def unconverted_set(self):
        return ParticlesMaskedSubset(self._private.particles.unconverted_set(), self._private.keys)

    def add_particles_to_store(self, keys, attributes = [], values = []):
        raise Exception("Cannot add particles to a masked subset")

    def remove_particles_from_store(self, keys):
        raise Exception("Cannot remove particles from a masked subset")


    def get_values_in_store(self, indices, attributes):
        if indices is None:
            indices = self.get_all_indices_in_store()
        return self._private.particles.get_values_in_store(indices, attributes)


        #subkeys = keys[~mask]
        #subvalues = self._private.particles.get_values_in_store(subkeys, attributes)

        #raise Exception("not implemented more of this")


    def set_values_in_store(self, indices, attributes, values):
        if indices is None:
            indices = self.get_all_indices_in_store()
            mask = self._private.keys.mask
        else:
            mask = indices.mask

        if numpy.all(~mask):
            return self._private.particles.set_values_in_store(indices, attributes, values)

        raise Exception("not implemented more of this")

    def get_attribute_names_defined_in_store(self):
        return self._private.particles.get_attribute_names_defined_in_store()

    def get_settable_attribute_names_defined_in_store(self):
        return self._private.particles.get_settable_attribute_names_defined_in_store()

    def get_all_keys_in_store(self):
        return self._private.keys

    def get_all_indices_in_store(self):
        if not self._private.version == self._private.particles._get_version():
            mask = self._private.keys.mask
            mask = ~mask
            indices = self._private.particles.get_indices_of_keys(self._private.keys[mask])
            self._private.indices = -1 * numpy.ones(len(self._private.keys), dtype = indices.dtype)
            self._private.indices = numpy.ma.array(self._private.indices, dtype = indices.dtype)
            self._private.indices.mask = self._private.keys.mask
            self._private.indices[mask] = indices
            self._private.version = self._private.particles._get_version()

        return self._private.indices

    def has_key_in_store(self, key):
        return key in self._private.set_of_keys

    def get_indices_of_keys(self, keys):
        if len(keys) == 1:
            return self._private.particles.get_indices_of_keys(keys)
        keys = numpy.ma.array(keys, dtype='uint64')
        result = -1 * numpy.ones(len(keys), dtype = numpy.int32)
        result[~keys.mask] = self._private.particles.get_indices_of_keys(keys[~keys.mask])
        return result

    def previous_state(self):
        return ParticlesMaskedSubset(self._private.particles.previous_state(), self._private.keys)


    def copy(self, memento = None, keep_structure = False, filter_attributes = lambda particle_set, x : True):
        attributes = self.get_attribute_names_defined_in_store()
        attributes = [x for x in attributes if filter_attributes(self, x)]

        keys = self.get_all_keys_in_store()
        keys = keys[~keys.mask]
        values = self.get_values_in_store(keys, attributes)
        result = Particles()
        converted = []
        if memento is None:
            memento = {}
        memento[id(self._original_set())] = result
        for x in values:
            if isinstance(x, LinkedArray):
                converted.append(x.copy(memento, keep_structure, filter_attributes))
            else:
                converted.append(x)
        result.add_particles_to_store(keys, attributes, converted)
        result._private.collection_attributes = self._private.collection_attributes._copy_for_collection(result)
        return result



    def copy_to_new_particles(self, keys = None, keys_generator = None, memento = None, keep_structure = False, filter_attributes = lambda particle_set, x : True):
        if keys_generator is None:
            keys_generator = UniqueKeyGenerator

        my_keys = self.get_all_keys_in_store()
        my_keys = my_keys[~my_keys.mask]

        if not keys is None:
            if len(keys) != len(my_keys):
                raise Exception('not enough new keys given for the copy')
            else:
                particle_keys = keys
        else:
            particle_keys = keys_generator.next_set_of_keys(len(my_keys))

        attributes = self.get_attribute_names_defined_in_store()
        attributes = [x for x in attributes if filter_attributes(self, x)]
        values = self.get_values_in_store(my_keys, attributes)
        result = Particles()
        if memento is None:
            memento = {}

        converted = []
        for x in values:
            if isinstance(x, LinkedArray):
                converted.append(x.copy(memento, keep_structure, filter_attributes))
            else:
                converted.append(x)

        memento[id(self._original_set())] = result

        result.add_particles_to_store(particle_keys, attributes, converted)

        result._private.collection_attributes = self._private.collection_attributes._copy_for_collection(result)

        return result

    def __str__(self):
        """
        Display string of a particle subset.

        >>> particles = Particles(keys=[1,2])
        >>> particles[0].child = particles[1]
        >>> print particles.child[0] # doctest: +ELLIPSIS
        Particle(2, set=<...>
            , child=None)
        >>> print particles.child[1]
        None
        >>> print particles[0] # doctest: +ELLIPSIS
        Particle(1, set=<...>
            , child=Particle(2, set=<...>))
        """
        keys = numpy.where(self.get_valid_particles_mask(), self._private.keys, [None])
        return str(list(keys))

    def as_set(self):
        return self


class ParticlesOverlay(AbstractParticleSet):
    """An overlay of of a particles set. The overlay
    stores extra attributes for the particles in the
    overlayed set.

    >>> p1 = Particles(3)
    >>> p1.mass = [10.0, 20.0, 30.0] | units.kg
    >>> p2 = ParticlesOverlay(p1)
    >>> p2.radius = [4.0, 5.0, 6.0] | units.m
    >>> print len(p2)
    3
    >>> print p2.mass
    [10.0, 20.0, 30.0] kg
    """

    def __init__(self, particles, overlay_set = None):
        AbstractParticleSet.__init__(self)

        self._private.base_set = particles
        if overlay_set is None:
            overlay_set = Particles(keys = self._private.base_set.key)
        self._private.overlay_set = overlay_set
        self._private.base_version = self._private.base_set._get_version()
        object.__setattr__(self, "_derived_attributes", particles._derived_attributes)

    def _ensure_updated_set_properties(self):
        if self._private.base_version == self._private.base_set._get_version():
            return

        self._private.base_version = self._private.base_set._get_version()

        base_set = self._private.base_set
        overlay_set = self._private.overlay_set

        base_set_keys = base_set.key
        overlay_set_keys = overlay_set.key
        removed_indices = []
        index = 0
        index_in_overlay = 0
        while index < len(base_set_keys) and index_in_overlay < len(overlay_set_keys):
            key1 = base_set_keys[index]
            key2 = overlay_set_keys[index_in_overlay]
            if key2 == key1:
                index_in_overlay += 1
                index += 1
            else:
                while key2 != key1:
                    removed_indices.append(index_in_overlay)
                    index_in_overlay += 1
                    if index_in_overlay >= len(overlay_set_keys):
                        break
                    key2 = overlay_set_keys[index_in_overlay]

        added_keys = []
        if index_in_overlay >= len(overlay_set_keys):
            added_keys = base_set_keys[index:]

        overlay_set.remove_particles_from_store(removed_indices)
        if len(added_keys) > 0:
            overlay_set.add_particles_to_store(added_keys)

    def can_extend_attributes(self):
        return self._private.overlay_set.can_extend_attributes()

    def __len__(self):
        return len(self._private.overlay_set)

    def _get_version(self):
        return self._private.overlay_set._get_version() +  self._private.base_set._get_version()


    def __getitem__(self, index):
        self._ensure_updated_set_properties()

        keys = self.get_all_keys_in_store()[index]

        if hasattr(keys, '__iter__'):
            return self._subset(keys)
        else:
            return Particle(keys, self)

    def _split_attributes(self, attributes):
        inbase = set(self._private.base_set.get_attribute_names_defined_in_store())
        inbase |= set(self._private.base_set._derived_attributes.keys())
        attributes_inbase = []
        attributes_inoverlay = []
        indices_inbase = []
        indices_inoverlay = []
        for i,x in enumerate(attributes):
            if x in inbase:
                attributes_inbase.append(x)
                indices_inbase.append(i)
            else:
                attributes_inoverlay.append(x)
                indices_inoverlay.append(i)
        return (attributes_inbase, indices_inbase), (attributes_inoverlay, indices_inoverlay)

    def _split_attributes_and_values(self, attributes, values, is_setter = True):
        if is_setter:
            inbase = set(self._private.base_set.get_settable_attribute_names_defined_in_store())
        else:
            inbase = set(self._private.base_set.get_attribute_names_defined_in_store())
        attributes_inbase = []
        attributes_inoverlay = []
        values_inbase = []
        values_inoverlay = []
        for x,y in zip(attributes, values):
            if x in inbase:
                attributes_inbase.append(x)
                values_inbase.append(y)
            else:
                attributes_inoverlay.append(x)
                values_inoverlay.append(y)
        return (attributes_inbase, values_inbase), (attributes_inoverlay, values_inoverlay)


    def add_particles_to_store(self, keys, attributes = [], values = []):
        self._ensure_updated_set_properties()

        (
            (attributes_inbase, values_inbase),
            (attributes_inoverlay, values_inoverlay)
        ) = self._split_attributes_and_values(attributes, values)


        self._private.base_set.add_particles_to_store(keys, attributes_inbase, values_inbase)
        self._private.overlay_set.add_particles_to_store(keys, attributes_inoverlay, values_inoverlay)

        self._private.base_version = self._private.base_set._get_version()

    def remove_particles_from_store(self, indices):
        self._ensure_updated_set_properties()

        indices = numpy.asarray(indices)

        self._private.base_set.remove_particles_from_store(indices[...,0])
        self._private.overlay_set.remove_particles_from_store(indices[...,1])

        self._private.base_version = self._private.base_set._get_version()



    def get_values_in_store(self, indices, attributes):
        self._ensure_updated_set_properties()

        (
            (attributes_inbase, indices_inbase),
            (attributes_inoverlay, indices_inoverlay)
        ) = self._split_attributes(attributes)


        if indices is None or isinstance(indices,EllipsisType):
            indices0 = self._private.base_set.get_all_indices_in_store()
            indices1 = self._private.overlay_set.get_all_indices_in_store()
        else:
            indices0 = []
            indices1 = []
            for i0, i1 in indices:
                indices0.append(i0)
                indices1.append(i1)
            indices0 = numpy.asarray(indices0, dtype='int64')
            indices1 = numpy.asarray(indices1, dtype='int64')

        result = [None] * len(attributes)
        if len(attributes_inbase) > 0:
            values_inbase = self._private.base_set.get_values_in_store(indices0, attributes_inbase)
            for i, value in zip(indices_inbase, values_inbase):
                result[i] = value

        if len(attributes_inoverlay) > 0:
            values_inoverlay = self._private.overlay_set.get_values_in_store(indices1, attributes_inoverlay)
            for i, value in zip(indices_inoverlay, values_inoverlay):
                result[i] = value

        return result

    def set_values_in_store(self, indices, attributes, values):
        self._ensure_updated_set_properties()

        (
            (attributes_inbase, values_inbase),
            (attributes_inoverlay, values_inoverlay)
        ) = self._split_attributes_and_values(attributes, values)


        if indices is None:
            indices0 = self._private.base_set.get_all_indices_in_store()
            indices1 = self._private.overlay_set.get_all_indices_in_store()
        else:
            indices0 = []
            indices1 = []
            for i0, i1 in indices:
                indices0.append(i0)
                indices1.append(i1)
            indices0 = numpy.asarray(indices0, dtype='int64')
            indices1 = numpy.asarray(indices1, dtype='int64')

        if len(attributes_inbase) > 0:
            self._private.base_set.set_values_in_store(indices0, attributes_inbase, values_inbase)
        if len(attributes_inoverlay) > 0:
            self._private.overlay_set.set_values_in_store(indices1, attributes_inoverlay, values_inoverlay)


    def get_attribute_names_defined_in_store(self):
        result = list(self._private.base_set.get_attribute_names_defined_in_store())
        result.extend(self._private.overlay_set.get_attribute_names_defined_in_store())
        return result

    def get_settable_attribute_names_defined_in_store(self):
        result = list(self._private.base_set.get_settable_attribute_names_defined_in_store())
        result.extend(self._private.overlay_set.get_settable_attribute_names_defined_in_store())
        return result

    def get_all_keys_in_store(self):
        self._ensure_updated_set_properties()

        return self._private.overlay_set.get_all_keys_in_store()

    def get_all_indices_in_store(self):
        self._ensure_updated_set_properties()

        indices0 = self._private.base_set.get_all_indices_in_store()
        indices1 = self._private.overlay_set.get_all_indices_in_store()

        return list(zip(indices0, indices1))

    def get_indices_of_keys(self, keys):
        self._ensure_updated_set_properties()

        indices0 = self._private.base_set.get_indices_of_keys(keys)
        indices1 = self._private.overlay_set.get_indices_of_keys(keys)

        return list(zip(indices0, indices1))

    def has_key_in_store(self, key):
        self._ensure_updated_set_properties()

        return self._private.overlay_set.has_key_in_store(key)

    def _original_set(self):
        return self
    
    
    def synchronize_to(self, other_particles):
        """
        Synchronize the particles of this set
        with the contents of the provided set.

        After this call the `other_particles` set will have
        the same particles as this set.

        This call will check if particles have been removed or
        added it will not copy values of existing particles
        over.

        :parameter other_particles: particle set wich has to be updated

        >>> particles = Particles(2)
        >>> particles.x = [1.0, 2.0] | units.m
        >>> copy = particles.copy()
        >>> new_particle = Particle()
        >>> new_particle.x = 3.0 | units.m
        >>> particles.add_particle(new_particle)# doctest:+ELLIPSIS
        <amuse.datamodel.particles.Particle object at ...>
        >>> print particles.x
        [1.0, 2.0, 3.0] m
        >>> print copy.x
        [1.0, 2.0] m
        >>> particles.synchronize_to(copy)
        >>> print copy.x
        [1.0, 2.0, 3.0] m

        """
        other_keys = set(other_particles.get_all_keys_in_store())
        my_keys = set(self.get_all_keys_in_store())
        added_keys = numpy.asarray(list(my_keys - other_keys))
        removed_keys = other_keys - my_keys
        removed_keys = list(removed_keys)
       
        if len(removed_keys) > 0:
            other_particles.remove_particles_from_store(other_particles.get_indices_of_keys(removed_keys))
            
        if len(added_keys) > 0:
            added_indices = self.get_indices_of_keys(added_keys)
                
            attributes = self.get_attribute_names_defined_in_store()
            attributes= [x for x in attributes if x not in other_particles._derived_attributes]
            values = self.get_values_in_store(added_indices, attributes)
            converted = []
            memento = {}

            for x in values:
                if isinstance(x, LinkedArray):
                    converted.append(x.copy_with_link_transfer(self._original_set(), other_particles, True, memento))
                    #converted.append(x.copy(memento, False))
                else:
                    converted.append(x)
            try:
                other_particles.add_particles_to_store(added_keys, attributes, converted)
            except exceptions.MissingAttributesAmuseException as caught_exception:
                for attribute_name in caught_exception.missing_attributes:
                    if attribute_name in self._derived_attributes:
                        attributes.append(attribute_name)
                        converted.append(getattr(self._subset(added_keys), attribute_name))
                    else:
                        raise
                other_particles.add_particles_to_store(added_keys, attributes, converted)



class ParticlesWithFilteredAttributes(AbstractParticleSet):
    """An overlaytof a particles set. The overlay
    filters attributes for the particles in the
    overlayed set.

    >>> p1 = Particles(3)
    >>> p1.mass = [10.0, 20.0, 30.0] | units.kg
    >>> p1.radius = [10.0, 20.0, 30.0] | units.m
    >>> p2 = ParticlesWithFilteredAttributes(p1, ["mass"])
    >>> print p2.get_attribute_names_defined_in_store()
    ['mass']
    >>> print p1.get_attribute_names_defined_in_store()
    ['mass', 'radius']
    """

    def __init__(self, particles, attributes_names, new_names = []):
        AbstractParticleSet.__init__(self)

        self._private.base_set = particles
        self._private.attribute_names = attributes_names
        if len(new_names) == 0:
            converted_names = attributes_names
        else:
            converted_names = []
            for set_name, converted_name in zip(attributes_names, new_names):
                if not converted_name:
                    converted_names.append(set_name)
                else:
                    converted_names.append(converted_name)

        mapping_from_set_name_to_converted_name = {}
        mapping_from_converted_name_to_set_name = {}
        for set_name, converted_name in zip(attributes_names, converted_names):
            mapping_from_set_name_to_converted_name[set_name] = converted_name
            mapping_from_converted_name_to_set_name[converted_name] = set_name

        self._private.mapping_from_set_name_to_converted_name = mapping_from_set_name_to_converted_name
        self._private.mapping_from_converted_name_to_set_name = mapping_from_converted_name_to_set_name
        self._private.converted_names = converted_names

    def _convert_to_base_set(self, attributes):
        result = []
        for x in attributes:
            if not x in self._private.mapping_from_converted_name_to_set_name:
                raise exceptions.AmuseException("attribute '{0}' not defined for this set".format(x))
            result.append(self._private.mapping_from_converted_name_to_set_name[x])
        return result

    def _filter_and_convert_to_this(self, attributes):
        return [self._private.mapping_from_set_name_to_converted_name[x] for x in attributes if x in self._private.mapping_from_set_name_to_converted_name]

    def can_extend_attributes(self):
        return False

    def __len__(self):
        return len(self._private.base_set)

    def _get_version(self):
        return self._private.base_set._get_version()

    def __getitem__(self, index):
        keys = self.get_all_keys_in_store()[index]

        if hasattr(keys, '__iter__'):
            return self._subset(keys)
        else:
            return Particle(keys, self)

    def add_particles_to_store(self, keys, attributes = [], values = []):
        attributes_inbase = self._convert_to_base_set(attributes)
        self._private.base_set.add_particles_to_store(keys, attributes_inbase, values)

    def remove_particles_from_store(self, indices):
        self._private.base_set.remove_particles_from_store(indices)

    def get_values_in_store(self, indices, attributes):
        attributes_inbase = self._convert_to_base_set(attributes)
        return self._private.base_set.get_values_in_store(indices, attributes_inbase)

    def set_values_in_store(self, indices, attributes, values):
        attributes_inbase = self._convert_to_base_set(attributes)
        self._private.base_set.set_values_in_store(indices, attributes_inbase, values)

    def get_attribute_names_defined_in_store(self):
        return self._private.converted_names

    def get_settable_attribute_names_defined_in_store(self):
        result = list(self._private.base_set.get_settable_attribute_names_defined_in_store())
        return self._filter_and_convert_to_this(result)

    def get_all_keys_in_store(self):
        return self._private.base_set.get_all_keys_in_store()

    def get_all_indices_in_store(self):
        return self._private.base_set.get_all_indices_in_store()

    def get_indices_of_keys(self, keys):
        return self._private.base_set.get_indices_of_keys(keys)

    def has_key_in_store(self, key):
        return self._private.base_set.has_key_in_store(key)

    def _original_set(self):
        return self

class ParticlesWithUnitsConverted(AbstractParticleSet):
    """
    A view on a particle sets. Used when to convert
    values between incompatible sets of units. For example to
    convert from si units to nbody units.

    The converter must have implement the ConverterInterface.

    >>> from amuse.units import nbody_system
    >>> particles_nbody = Particles(2)
    >>> particles_nbody.x = [10.0 , 20.0] | nbody_system.length
    >>> convert_nbody = nbody_system.nbody_to_si(10 | units.kg , 5 | units.m )
    >>> particles_si = ParticlesWithUnitsConverted(
    ...     particles_nbody,
    ...     convert_nbody.as_converter_from_si_to_nbody())
    ...
    >>> print particles_nbody.x
    [10.0, 20.0] length
    >>> print particles_si.x
    [50.0, 100.0] m
    >>> particles_si.x = [200.0, 400.0] | units.m
    >>> print particles_nbody.x
    [40.0, 80.0] length


    """

    class ConverterInterface(object):
        """
        Interface definition for the converter.

        source
            The source quantity is in the units of the user of a
            ParticlesWithUnitsConverted object
        target
            The target quantity must be in the units of the
            internal particles set.

        """
        def from_source_to_target(quantity):
            """
            Converts the quantity from the source units
            to the target units.

            :parameter quantity: quantity to convert
            """
            return quantity


        def from_target_to_source(quantity):
            """
            Converts the quantity from the target units
            to the source units.

            :parameter quantity: quantity to convert
            """
            return quantity


    def __init__(self, particles, converter):
        AbstractParticleSet.__init__(self, particles)

        self._private.particles = particles
        self._private.converter = converter

    def compressed(self):
        return ParticlesWithUnitsConverted(self._private.particles.compressed(), self._private.converter)

    def get_valid_particles_mask(self):
        return self._private.particles.get_valid_particles_mask()

    def __getitem__(self, index):

        keys = self.get_all_keys_in_store()[index]

        if keys is ma.masked:
            return None
        elif hasattr(keys, '__iter__'):
            return self._subset(keys)
        else:
            return Particle(keys, self)

    def _get_version(self):
        return self._private.particles._get_version()

    def shallow_copy(self):
        copiedParticles =  self._private.particles.shallow_copy()
        return ParticlesWithUnitsConverted(copiedParticles, self._private.converter)

    def unconverted_set(self):
        return self._private.particles


    def can_extend_attributes(self):
        return self._private.particles.can_extend_attributes()

    def add_particles_to_store(self, keys, attributes = [], values = []):
        converted_values = []
        for quantity in values:
            converted_quantity = self._private.converter.from_source_to_target(quantity)
            converted_values.append(converted_quantity)
        self._private.particles.add_particles_to_store(keys, attributes, converted_values)

    def remove_particles_from_store(self, keys):
        self._private.particles.remove_particles_from_store(keys)

    def get_values_in_store(self, indices, attributes):
        values = self._private.particles.get_values_in_store(indices, attributes)
        converted_values = []
        for quantity in values:
            if isinstance(quantity, LinkedArray):
                objects = quantity
                convert_objects = []
                for x in objects:
                    if x is None:
                        convert_objects.append(x)
                    else:
                        if isinstance(x, Particle):
                            convert_objects.append(ParticlesWithUnitsConverted(x.as_set(), self._private.converter)[0])
                        else:
                            convert_objects.append(ParticlesWithUnitsConverted(x, self._private.converter))
                convert_objects = LinkedArray(convert_objects)
                converted_values.append(convert_objects)
            else:
                converted_quantity = self._private.converter.from_target_to_source(quantity)
                converted_values.append(converted_quantity)
        return converted_values

    def set_values_in_store(self, indices, attributes, values):
        converted_values = []
        for quantity in values:
            converted_quantity = self._private.converter.from_source_to_target(quantity)
            converted_values.append(converted_quantity)
        self._private.particles.set_values_in_store(indices, attributes, converted_values)

    def get_attribute_names_defined_in_store(self):
        return self._private.particles.get_attribute_names_defined_in_store()

    def get_settable_attribute_names_defined_in_store(self):
        return self._private.particles.get_settable_attribute_names_defined_in_store()

    def get_all_keys_in_store(self):
        return self._private.particles.get_all_keys_in_store()

    def get_all_indices_in_store(self):
        return self._private.particles.get_all_indices_in_store()

    def get_indices_of_keys(self, keys):
        return self._private.particles.get_indices_of_keys(keys)

    def has_key_in_store(self, key):
        return self._private.particles.has_key_in_store(key)

    def as_set(self):
        return ParticlesSubset(self, self.get_all_keys_in_store())

    def get_timestamp(self):
        timestamp = self._private.particles.get_timestamp()
        if not timestamp is None:
            timestamp = self._private.converter.from_target_to_source(timestamp)
        return timestamp

    def savepointsavepoint(self, timestamp=None):
        if not timestamp is None:
            timestamp = self._private.converter.from_target_to_source(timestamp)
        return ParticlesWithUnitsConverted(
            self._private.particles.savepoint(timestamp),
            self._private.converter
        )

    def previous_state(self):
        return ParticlesWithUnitsConverted(
            self._private.particles.previous_state(),
            self._private.converter
        )

    def get_unit_converter(self):
        return self._private.converter.generic_to_si



class ParticlesWithAttributesTransformed(AbstractParticleSet):
    """
    A view on a particle sets. Some attributes are transformed
    on input or output.
    """

    def __init__(self, particles, get_function, set_function):
        AbstractParticleSet.__init__(self, particles)

        self._private.particles = particles
        self._private.get_function = get_function
        self._private.set_function = set_function

    @classmethod
    def translate(cls, particles, position, velocity):
        def get_function(attribute, quantity):
            if(attribute == 'x'):
                return quantity + position[0]
            elif(attribute == 'y'):
                return quantity + position[1]
            elif(attribute == 'z'):
                return quantity + position[2]
            elif(attribute == 'vx'):
                return quantity + velocity[0]
            elif(attribute == 'vy'):
                return quantity + velocity[1]
            elif(attribute == 'vz'):
                return quantity + velocity[2]
            else:
                return quantity

        def set_function(attribute, quantity):
            if(attribute == 'x'):
                return quantity - position[0]
            elif(attribute == 'y'):
                return quantity - position[1]
            elif(attribute == 'z'):
                return quantity - position[2]
            elif(attribute == 'vx'):
                return quantity - velocity[0]
            elif(attribute == 'vy'):
                return quantity - velocity[1]
            elif(attribute == 'vz'):
                return quantity - velocity[2]
            else:
                return quantity



        return cls(
            particles,
            get_function,
            set_function
        )

    def compressed(self):
        return ParticlesWithAttributesTransformed(
            self._private.particles.compressed(),
            self._private.get_function,
            self._private.set_function
        )

    def get_valid_particles_mask(self):
        return self._private.particles.get_valid_particles_mask()

    def __getitem__(self, index):

        keys = self.get_all_keys_in_store()[index]

        if keys is ma.masked:
            return None
        elif hasattr(keys, '__iter__'):
            return self._subset(keys)
        else:
            return Particle(keys, self)

    def _get_version(self):
        return self._private.particles._get_version()

    def shallow_copy(self):
        copiedParticles =  self._private.particles.shallow_copy()
        return ParticlesWithAttributesTransformed(
            copiedParticles,
            self._private.get_function,
            self._private.set_function,
        )

    def unconverted_set(self):
        return self._private.particles

    def can_extend_attributes(self):
        return self._private.particles.can_extend_attributes()

    def add_particles_to_store(self, keys, attributes = [], values = []):
        converted_values = []
        for attribute, quantity in zip(attributes, values):
            converted_quantity = self._private.set_function(attribute, quantity)
            converted_values.append(converted_quantity)
        self._private.particles.add_particles_to_store(keys, attributes, converted_values)

    def remove_particles_from_store(self, keys):
        self._private.particles.remove_particles_from_store(keys)

    def get_values_in_store(self, indices, attributes):
        values = self._private.particles.get_values_in_store(indices, attributes)
        converted_values = []
        for attribute, quantity in zip(attributes, values):
            if isinstance(quantity, LinkedArray):
                objects = quantity
                convert_objects = []
                for x in objects:
                    if x is None:
                        convert_objects.append(x)
                    else:
                        if isinstance(x, Particle):
                            convert_objects.append(ParticlesWithAttributesTransformed(x.as_set(), self._private.get_function, self._private.set_function)[0])
                        else:
                            convert_objects.append(ParticlesWithAttributesTransformed(x, self._private.get_function, self._private.set_function))
                convert_objects = LinkedArray(convert_objects)
                converted_values.append(convert_objects)
            else:
                converted_quantity = self._private.get_function(attribute, quantity)
                converted_values.append(converted_quantity)
        return converted_values


    def set_values_in_store(self, indices, attributes, values):
        converted_values = []
        for attribute, quantity in zip(attributes, values):
            converted_quantity = self._private.set_function(attribute, quantity)
            converted_values.append(converted_quantity)
        self._private.particles.set_values_in_store(indices, attributes, converted_values)

    def get_attribute_names_defined_in_store(self):
        return self._private.particles.get_attribute_names_defined_in_store()

    def get_settable_attribute_names_defined_in_store(self):
        return self._private.particles.get_settable_attribute_names_defined_in_store()

    def get_all_keys_in_store(self):
        return self._private.particles.get_all_keys_in_store()

    def get_all_indices_in_store(self):
        return self._private.particles.get_all_indices_in_store()

    def get_indices_of_keys(self, keys):
        return self._private.particles.get_indices_of_keys(keys)

    def has_key_in_store(self, key):
        return self._private.particles.has_key_in_store(key)

    def as_set(self):
        return ParticlesSubset(self, self.get_all_keys_in_store())

    def get_timestamp(self):
        timestamp = self._private.particles.get_timestamp()
        if not timestamp is None:
            timestamp = self._private.get_function("timestamp", timestamp)
        return timestamp

    def savepoint(self, timestamp=None, **kwargs):
        if not timestamp is None:
            timestamp = self._private.set_function("timestamp", timestamp)
        return ParticlesWithAttributesTransformed(
            self._private.particles.savepoint(timestamp),
            self._private.get_function,
            self._private.set_function,
        )

    def previous_state(self):
        return ParticlesWithAttributesTransformed(
            self._private.particles.previous_state(),
            self._private.get_function,
            self._private.set_function,
        )

class TransformedParticles(ParticlesWithAttributesTransformed):
    """
    Particleset which is a view on the parent particle set with some attributes transformed.
    
    This is a more flexible reimplementation of ParticlesWithAttributesTransformed such
    that general coordinate trasnformations are possible.
    """

    @classmethod
    def translate(cls, particles, position, velocity):
        def forward(x,y,z,vx,vy,vz):
            return [x+position[0],
                    y+position[1],
                    z+position[2],
                    vx+velocity[0],
                    vy+velocity[1],
                    vz+velocity[2],]

        def reverse(x,y,z,vx,vy,vz):
            return [x-position[0],
                    y-position[1],
                    z-position[2],
                    vx-velocity[0],
                    vy-velocity[1],
                    vz-velocity[2],]

        return cls(
            particles,
            ["x","y","z","vx","vy","vz"],
            forward,
            ["x","y","z","vx","vy","vz"],
            reverse
        )

    @classmethod
    def rotate_z(cls, particles, angle, omega):
        def forward(x,y,vx,vy, inverse=False):
            _angle=angle
            _omega=omega
            if inverse:
                _angle=-angle
                _omega=-omega
    
            C1 = vx + _omega*y
            C2 = vy - _omega*x
            x_ =  x * trigo.cos(_angle) + y * trigo.sin(_angle)
            y_ = -x * trigo.sin(_angle) + y * trigo.cos(_angle)
            vx_ = C1*trigo.cos(_angle) + C2*trigo.sin(_angle)
            vy_ = C2*trigo.cos(_angle) - C1*trigo.sin(_angle)
            return x_,y_,vx_,vy_

        def reverse(x,y,vx,vy):
            return forward(x,y,vx,vy, inverse=True)

        return cls(
            particles,
            ["x","y","vx","vy"],
            forward,
            ["x","y","vx","vy"],
            reverse
        )

    def __init__(self, particles, target_attributes, 
        forward_transformation, source_attributes, reverse_transformation=None):
        AbstractParticleSet.__init__(self, particles)

        self._private.particles = particles
        self._private.source_attributes=source_attributes
        self._private.target_attributes=target_attributes
        self._private.forward_transformation = forward_transformation
        self._private.reverse_transformation = reverse_transformation
        if forward_transformation is None:
            self._private.forward_transformation = lambda *x:x
            self._private.reverse_transformation = lambda *x:x
          

    def compressed(self):
        return TransformedParticles(
            self._private.particles.compressed(),
            self._private.source_attributes,
            self._private.target_attributes,
            self._private.forward_transformation,
            self._private.reverse_transformation,
        )

    def shallow_copy(self):
        copiedParticles =  self._private.particles.shallow_copy()
        return TransformedParticles(
            copiedParticles,
            self._private.source_attributes,
            self._private.target_attributes,
            self._private.forward_transformation,
            self._private.reverse_transformation,
        )


    def get_values_in_store(self, indices, attributes):
        
        direct_attributes=[]
        needs_transformation=False
        for attribute in attributes:
            if attribute in self._private.target_attributes:
                needs_transformation=True
            else:
                direct_attributes.append(attribute)
                
        if needs_transformation:
            sources=self._private.particles.get_values_in_store(indices, self._private.source_attributes)
            targets=self._private.forward_transformation(*sources)
        
        if direct_attributes:
            direct_values = self._private.particles.get_values_in_store(indices, direct_attributes)

        values=[]
        for attribute in attributes:
            if attribute in self._private.target_attributes:
                value=targets[self._private.target_attributes.index(attribute)]
            else:
                value=direct_values[direct_attributes.index(attribute)]
            values.append(value)

        converted_values = []
        for attribute, quantity in zip(attributes, values):
            if isinstance(quantity, LinkedArray):
                objects = quantity
                convert_objects = []
                for x in objects:
                    if x is None:
                        convert_objects.append(x)
                    else:
                        transformed_x=TransformedParticles(
                                x.as_set(),
                                self._private.source_attributes,
                                self._private.target_attributes,
                                self._private.forward_transformation,
                                self._private.reverse_transformation,
        )
                        if isinstance(x, Particle):
                            transformed_x=x[0]
                        convert_objects.append(transformed_x)
                convert_objects = LinkedArray(convert_objects)
                converted_values.append(convert_objects)
            else:
                converted_values.append(quantity)
        return converted_values


    def set_values_in_store(self, indices, attributes, values):
        _attributes=[]
        _values=[]
        needs_transformation=False
        for attribute, value in zip(attributes, values):
            if attribute in self._private.target_attributes:
                needs_transformation=True
            else:
                _attributes.append(attribute)
                _values.append(value)

        if needs_transformation:
            if self._private.reverse_transformation is None:
                raise Exception("no reverse_transformation defined, view on particleset is readonly")
            missing=[]
            for attribute in self._private.target_attributes:
                if attribute not in attributes:
                    missing.append(attribute)

            if missing:
                sources=self._private.particles.get_values_in_store(indices, self._private.source_attributes)
                targets=self._private.forward_transformation(*sources)
            
            args=[]    
            for attribute in self._private.target_attributes:
                if attribute in missing:
                    value=targets[self._private.target_attributes.index(attribute)]
                else:
                    value=values[attributes.index(attribute)]
                args.append(value)
                
            sources=self._private.reverse_transformation(*args)
            for attribute,value in zip(self._private.source_attributes, sources):
                _attributes.append(attribute)
                _values.append(value)

        self._private.particles.set_values_in_store(indices, _attributes, _values)

    def add_particles_to_store(self, keys, attributes = [], values = []):
        _attributes=[]
        _values=[]
        needs_transformation=False
        for attribute, value in zip(attributes, values):
            if attribute in self._private.target_attributes:
                needs_transformation=True
            else:
                _attributes.append(attribute)
                _values.append(value)

        if needs_transformation:
            if self._private.reverse_transformation is None:
                raise Exception("no reverse_transformation defined, view on particleset is readonly")
            missing=[]
            for attribute in self._private.target_attributes:
                if attribute not in attributes:
                    missing.append(attribute)

            if missing:
                raise Exception("For this add_particle operation I am missing the following attributes:"+ str(missing))
            
            args=[]    
            for attribute in self._private.target_attributes:
                value=values[attributes.index(attribute)]
                args.append(value)
                
            sources=self._private.reverse_transformation(*args)
            for attribute,value in zip(self._private.source_attributes, sources):
                _attributes.append(attribute)
                _values.append(value)
        self._private.particles.add_particles_to_store(keys, _attributes, _values)

    def get_timestamp(self):
        return self._private.particles.get_timestamp()

    def savepoint(self, timestamp=None, **kwargs):
        return TransformedParticles(
            self._private.particles.savepoint(timestamp),
            self._private.source_attributes,
            self._private.target_attributes,
            self._private.forward_transformation,
            self._private.reverse_transformation,
        )


    def previous_state(self):
        return TransformedParticles(
            self._private.particles.previous_state(),
            self._private.source_attributes,
            self._private.target_attributes,
            self._private.forward_transformation,
            self._private.reverse_transformation,
        )


class ParticleInformationChannel(object):

    def __init__(self, from_particles, to_particles, attributes=None, target_names=None):
        self.from_particles = from_particles
        self.to_particles = to_particles
        self.attributes = attributes
        self.target_names = target_names
        self.from_version = -1
        self.to_version = -1
        self._reindex()

    def _reindex(self):
        if (
            (self.from_version == self.from_particles._get_version()) and
            (self.to_version == self.to_particles._get_version())
           ):
            return


        self.keys = self.intersecting_keys()
        self.from_indices = self.from_particles.get_indices_of_keys(self.keys)
        self.to_indices = self.to_particles.get_indices_of_keys(self.keys)
        self.from_version = self.from_particles._get_version()
        self.to_version = self.to_particles._get_version()


    def reverse(self):
        if self.target_names is None:
            attributes = self.attributes
            target_names = self.target_names
        else:
            attributes = self.target_names
            target_names = self.attributes

        return ParticleInformationChannel(
            self.to_particles,
            self.from_particles,
            attributes,
            target_names
        )

    def intersecting_keys(self):
        from_keys = self.from_particles.get_all_keys_in_store()
        to_keys = self.to_particles.get_all_keys_in_store()
        return numpy.intersect1d(from_keys,to_keys) #filter(lambda x : self.to_particles._has_key(x), from_keys)

    def copy_attributes(self, attributes, target_names = None):
        if target_names is None:
            target_names = attributes

        self._reindex()

        if len(self.to_indices) == 0:
            return

        if len(self.from_indices) == 0:
            return
        
        if len(self.keys) == 0:
            return
        
        values = self.from_particles.get_values_in_store(self.from_indices, attributes)
        
        converted = []
        for x in values:
            if isinstance(x, LinkedArray):
                converted.append(x.copy_with_link_transfer(self.from_particles, self.to_particles))
            else:
                converted.append(x)
        self.to_particles.set_values_in_store(self.to_indices, target_names, converted)

    def copy_attributes_async(self, attributes, target_names = None, async_get = True, async_set = False):

        if target_names is None:
            target_names = attributes

        self._reindex()

        if len(self.to_indices) == 0:
            return

        if len(self.from_indices) == 0:
            return
        
        if len(self.keys) == 0:
            return
       
        if async_get:
            request = self.from_particles.get_values_in_store_async(self.from_indices, attributes)
            def result_handler(inner):
                values = inner()
                converted = []
                for x in values:
                    if isinstance(x, LinkedArray):
                        converted.append(x.copy_with_link_transfer(self.from_particles, self.to_particles))
                    else:
                        converted.append(x)
                self.to_particles.set_values_in_store(self.to_indices, target_names, converted)
                return converted
            request.add_result_handler(result_handler)
            return request
        elif async_set:
            values = self.from_particles.get_values_in_store(self.from_indices, attributes)
            
            converted = []
            for x in values:
                if isinstance(x, LinkedArray):
                    converted.append(x.copy_with_link_transfer(self.from_particles, self.to_particles))
                else:
                    converted.append(x)
            request = self.to_particles.set_values_in_store_async(self.to_indices, target_names, converted)
            return request





    def copy(self):
        if not self.attributes is None:
            self.copy_attributes(self.attributes, self.target_names)
        elif not self.to_particles.can_extend_attributes():
            self.copy_overlapping_attributes()
        else:
            self.copy_all_attributes()

    def copy_all_attributes(self):
        names_to_copy = self.from_particles.get_attribute_names_defined_in_store()
        self.copy_attributes(list(names_to_copy))


    def copy_overlapping_attributes(self):
        from_names = self.from_particles.get_attribute_names_defined_in_store()
        to_names = self.to_particles.get_settable_attribute_names_defined_in_store()
        names_to_copy = set(from_names).intersection(set(to_names))

        self.copy_attributes(list(names_to_copy))


    def copy_attribute(self, name, target_name = None):
        """ Copy the values of one attribute from the source set to the target set.
        The copied attribute can have a different name in the target set.

        :argument name: name of the attribute in the source set
        :argument target_name: name of the attribute in the target set, when None (default) the target_name
           will be set equal to the name

        >>> from amuse.datamodel import Particles
        >>> from amuse.units import units
        >>> particles1 = Particles(2)
        >>> particles2 = particles1.copy()
        >>> particles1.mass = 1 | units.m
        >>> particles2.mass = 3 | units.m
        >>> channel = particles1.new_channel_to(particles2)
        >>> channel.copy_attribute("mass", "mass_from_p2")
        >>> print particles2.mass_from_p2
        [1.0, 1.0] m
        >>> print particles2.mass - particles2.mass_from_p2
        [2.0, 2.0] m

        """
        if target_name is None:
            target_name = name

        self._reindex()
        
        if len(self.keys) == 0:
            return
            
        data = self.from_particles.get_values_in_store(self.from_indices, [name,])
        self.to_particles.set_values_in_store(self.to_indices, [target_name,], data)

    def transform_values(self, attributes, f):
        values = self.from_particles.get_values_in_store(self.from_indices, attributes)
        return f(*values)

    def transform(self, target, function, source):
        """ Copy and transform values of one attribute from the source set to the target set.

        :argument target: name of the attributes in the target set
        :argument function: function used for transform, should return tuple 
        :argument source: name of the attribute in the source set

        >>> from amuse.datamodel import Particles
        >>> particles1 = Particles(3)
        >>> particles2 = particles1.copy()
        >>> particles1.attribute1 = 1
        >>> particles1.attribute2 = 2
        >>> channel = particles1.new_channel_to(particles2)
        >>> channel.transform(["attribute3"], lambda x,y: (x+y,), ["attribute1","attribute2"])
        >>> print particles2.attribute3
        [3 3 3]
        >>> channel.transform(["attribute1","attribute1b"], lambda x: (x,2*x), ["attribute1"])
        >>> print particles2.attribute1b
        [2 2 2]

        """
        self._reindex()
        
        if len(self.keys) == 0:
            return

        if function is None:
            function=lambda *x : x

        if not self.to_particles.can_extend_attributes():
            target_attributes = self.to_particles.get_defined_settable_attribute_names()
            if not set(target).issubset(set(target_attributes)):
                raise Exception("trying to set unsettable attributes {0}".format(
                                list(set(target)-set(target_attributes))) )
        converted=self.transform_values(source, function)
        if len(converted) != len(target):
            raise Exception("function {0} returns {1} values while target attributes are {2} of length {3}".format(
                            function.__name__, len(converted), target, len(target)))
        self.to_particles.set_values_in_store(self.to_indices, target, converted)        


class Channels(object):
    def __init__(self, channels=None):
        self._channels = []
        if channels is not None:
            self.add_channels(channels)

    def add_channel(self, channel):
        self._channels.append(channel)

    def add_channels(self, channels):
            if isinstance(channels, Channels):
                self._channels += channels._channels
            else:
                for chan in iter(channels):
                    self.add_channel(chan)

    def remove_channel(self, channel):
        self._channels.remove(channel)

    def copy(self):
        for channel in self._channels:
            channel.copy()

class ParticlesWithNamespacedAttributesView(AbstractParticleSet):
    """
    A view on prefixed attributes of a particle set.
    """

    def __init__(self, particles, namespace):
        AbstractParticleSet.__init__(self, particles)

        self._private.particles = particles
        self._private.namespace = namespace

    def compressed(self):
        return ParticlesWithNamespacedAttributesView(
            self._private.particles.compressed(),
            self._private.namespace
        )

    def get_valid_particles_mask(self):
        return self._private.particles.get_valid_particles_mask()

    def __getitem__(self, index):

        keys = self.get_all_keys_in_store()[index]

        if keys is ma.masked:
            return None
        elif hasattr(keys, '__iter__'):
            return self._subset(keys)
        else:
            return Particle(keys, self)

    def _get_version(self):
        return self._private.particles._get_version()

    def shallow_copy(self):
        copiedParticles =  self._private.particles.shallow_copy()
        return ParticlesWithNamespacedAttributesView(
            copiedParticles,
            self._private.namespace
        )

    def unconverted_set(self):
        return self._private.particles

    def can_extend_attributes(self):
        return self._private.particles.can_extend_attributes()

    def add_particles_to_store(self, keys, attributes = [], values = []):
        namespace = self._private.namespace
        namespaced_attributes = [namespace + '__' + x for x in attributes]
        self._private.particles.add_particles_to_store(keys, namespaced_attributes, values)

    def remove_particles_from_store(self, keys):
        self._private.particles.remove_particles_from_store(keys)

    def get_values_in_store(self, indices, attributes):
        namespace = self._private.namespace
        namespaced_attributes = [namespace + '__' + x for x in attributes]
        return self._private.particles.get_values_in_store(indices, namespaced_attributes)


    def set_values_in_store(self, indices, attributes, values):
        namespace = self._private.namespace
        namespaced_attributes = [namespace + '__' + x for x in attributes]
        self._private.particles.set_values_in_store(indices, namespaced_attributes, values)

    def get_attribute_names_defined_in_store(self):
        names = self._private.particles.get_attribute_names_defined_in_store()
        namespace_prefix = self._private.namespace  + '__'
        len_namespace_prefix = len(namespace_prefix)
        return [x[len_namespace_prefix:] for x in names if x.startswith(namespace_prefix)]


    def get_settable_attribute_names_defined_in_store(self):
        names = self._private.particles.get_settable_attribute_names_defined_in_store()
        namespace_prefix = self._private.namespace  + '__'
        len_namespace_prefix = len(namespace_prefix)
        return [x[len_namespace_prefix:] for x in names if x.startswith(namespace_prefix)]

    def get_all_keys_in_store(self):
        return self._private.particles.get_all_keys_in_store()

    def get_all_indices_in_store(self):
        return self._private.particles.get_all_indices_in_store()

    def get_indices_of_keys(self, keys):
        return self._private.particles.get_indices_of_keys(keys)

    def has_key_in_store(self, key):
        return self._private.particles.has_key_in_store(key)

    def as_set(self):
        return ParticlesSubset(self, self.get_all_keys_in_store())

    def get_timestamp(self):
        return self._private.particles.get_timestamp()

    def savepoint(self, timestamp=None):
        return ParticlesWithNamespacedAttributesView(
            self._private.particles.savepoint(timestamp),
            self._private.namespace
        )

    def previous_state(self):
        return ParticlesWithNamespacedAttributesView(
            self._private.particles.previous_state(),
            self._private.namespace
        )



class DomainAttribute(DerivedAttribute):
    """
    Combine multiple attributes into the same namespace
    """
    def  __init__(self, name):
        self.name = name

    def get_values_for_entities(self, instance):
        return ParticlesWithNamespacedAttributesView(instance, self.name)

    def set_values_for_entities(self, instance, value):
        raise AttributeError('"{0}" is already defined as a namespace attribute, you cannot assign a value to it'.format(self.name))

    def get_value_for_entity(self, instance, particle, index):
        namespaced_set = ParticlesWithNamespacedAttributesView(particle.particles_set, self.name)
        # or:
        # return namespaced_set[index]
        return Particle(
            particle.key,
            namespaced_set,
            particle._set_index,
            particle._set_version
        )

    def set_value_for_entity(self, instance, key, vector):
        raise AttributeError('"{0}" is already defined as a namespace attribute, you cannot assign a value to it'.format(self.name))




class Stars(Particles):
    pass

class Particle(object):
    """A physical object or a physical region simulated as a
    physical object (cloud particle).

    All attributes defined on a particle are specific for
    that particle (for example mass or position). A particle contains
    a set of attributes, some attributes are *generic* and applicable
    for multiple modules. Other attributes are *specific* and are
    only applicable for a single module.


    """
    __slots__ = ("key", "particles_set", "_set_index", "_set_version")

    # these are defined so that numpy conversion is way faster
    # otherwhise it would go through the __getattr__ function
    # which will slow it down by a factor 3
    if compare_version_strings(numpy.__version__, '1.7.0') < 0:
        __array_interface__ = {'shape':() }
    else:
        __array_interface__ = {'shape':(),'typestr':'|O4' }

    def __len__(self):
        raise AttributeError()
    def __iter__(self):
        raise AttributeError()

    __array_struct__ = UndefinedAttribute()
    __array__ = UndefinedAttribute()

    def __init__(self, key = None, particles_set = None, set_index = None, set_version = -1, **keyword_arguments):
        if particles_set is None:
            if key == None:
                particles_set = Particles(1)
                key = particles_set.get_all_keys_in_store()[0]
            else:
                particles_set = Particles(1, keys = [key])

        object.__setattr__(self, "key", key)
        object.__setattr__(self, "particles_set", particles_set)
        object.__setattr__(self, "_set_index", set_index)
        object.__setattr__(self, "_set_version", set_version)

        for attribute_name in keyword_arguments:
            attribute_value = keyword_arguments[attribute_name]
            setattr(self, attribute_name, attribute_value)

    def __getstate__(self):
        return (self.key, self.as_set().copy())

    def __setstate__(self, key_and_set):
        key, particles_set = key_and_set
        object.__setattr__(self, "key", key)
        object.__setattr__(self, "particles_set", particles_set)
        object.__setattr__(self, "_set_index", None)
        object.__setattr__(self, "_set_version", None)


    def __setattr__(self, name_of_the_attribute, new_value_for_the_attribute):
        if self._set_index is None or self._set_version != self.particles_set._get_version():
            object.__setattr__(self, "_set_index", self.particles_set.get_indices_of_keys([self.key])[0])
            object.__setattr__(self, "_set_version", self.particles_set._get_version())

        self.particles_set._set_value_of_attribute(
            self._set_index,
            name_of_the_attribute,
            new_value_for_the_attribute
        )


    def __getattr__(self, name_of_the_attribute):
        if self._set_index is None or self._set_version != self.particles_set._get_version():
            object.__setattr__(self, "_set_index", self.particles_set.get_indices_of_keys([self.key])[0])
            object.__setattr__(self, "_set_version", self.particles_set._get_version())
        try:
            return self.particles_set._get_value_of_attribute(self, self._set_index, name_of_the_attribute)
        except Exception as ex:
            raise AttributeError("You tried to access attribute '{0}' but this attribute is not defined for this set.".format(name_of_the_attribute, ex))

    def children(self):
        return self.particles_set.select(lambda x : x == self, ["parent"])

    def descendents(self):
        result = self.children()
        stack = list(result)
        while len(stack) > 0:
            current = stack.pop()
            children = current.children()
            result = result.union(children)
            stack.extend(children)
        return result

    def add_child(self, child):
        if self.particles_set != child.particles_set:
            raise exceptions.AmuseException("The parent and child particles should be in the same set")

        child.parent = self


    def copy(self):
        return self.particles_set.copy()._get_particle(self.key)

    def empty_copy(self):
        keys = [self.key]
        result = Particles()
        result.add_particles_to_store(keys, [],[])
        object.__setattr__(result, "_derived_attributes", CompositeDictionary(self.particles_set._derived_attributes))
        return result._get_particle(self.key)

    def __add__(self, particles):
        """
        Returns a particle subset, composed of the given
        particle(s) and this particle. Attribute values are
        not stored by the subset. The subset provides a view
        on the particles.

        :parameter particles: particle(s) to be added to self.

        >>> particles = Particles(2)
        >>> particle1 = particles[0]
        >>> particle1.x = 1.0 | units.m
        >>> particle2 = particles[1]
        >>> particle2.x = 2.0 | units.m
        >>> new_set = particle1 + particle2
        >>> new_set  # doctest:+ELLIPSIS
        <amuse.datamodel.particles.ParticlesSubset object at 0x...>
        >>> print len(new_set)
        2
        >>> print new_set.x
        [1.0, 2.0] m
        """
        return self.as_set().__add__(particles)

    def __sub__(self, particles):
        """
        Raises an exception: cannot subtract particle(s)
        from a particle.
        """
        raise exceptions.AmuseException("Cannot subtract particle(s) from a particle.")

    def __str__(self):
        """
        Display string for a particle

        >>> p = Particle(10)
        >>> p.x = 10.2 | units.m
        >>> p.mass = 5 | units.kg
        >>> print p # doctest: +ELLIPSIS
        Particle(10, set=<...>
            , mass=5.0 kg
            , x=10.2 m)
        """
        if self._set_index is None or self._set_version != self.particles_set._get_version():
            object.__setattr__(self, "_set_index", self.particles_set.get_indices_of_keys([self.key])[0])
            object.__setattr__(self, "_set_version", self.particles_set._get_version())

        output = 'Particle('
        output += str(self.key)
        output += ', set=<{0}>'.format(id(self.particles_set))
        
        for name, value in self.particles_set._values_of_particle(self._set_index):
            output += '\n    , '
            output += name
            output += '='
            if isinstance(value, Particle):
                output += 'Particle('
                output += str(value.key)
                output += ', set=<{0}>'.format(id(value.particles_set))
                output += ')'
            elif isinstance(value, AbstractParticleSet):
                output += value.__class__.__name__
                output += '(len={0}, id={1})'.format(len(value), id(value))
            else:
                output += str(value)
        output += ')'
        return output



    def __dir__(self):
        result = []
        result.extend(dir(type(self)))
        result.extend(self.particles_set._attributes_for_dir())
        return result

    def __eq__(self, other):
        return isinstance(other, type(self)) and other.key == self.key

    def __hash__(self):
        return self.key.__hash__()

    def __ne__(self, other):
        return not (isinstance(other, type(self)) and other.key == self.key)

    def set_default(self, attribute, quantity):
        if not attribute in self.particles_set.get_attribute_names_defined_in_store():
            self.particles_set._set_value_of_attribute(self, attribute, quantity)

    def get_timeline_of_attribute(self, attribute):
        return self.particles_set.get_timeline_of_attribute(self.key, attribute)

    def get_timeline_of_attribute_as_vector(self, attribute):
        return self.particles_set.get_timeline_of_attribute_as_vector(self.key, attribute)

    def get_timeline_of_attributes(self, attributes):
        return self.particles_set.get_timeline_of_attributes(self.key, attributes)

    def as_set(self):
        """
        Returns a subset view on the set containing this particle. The
        subset view includes this particle and no other particles.

        >>> particles = Particles(2)
        >>> particles.x = [1.0, 2.0] | units.m
        >>> particle2 = particles[1]
        >>> print particle2.x
        2.0 m
        >>> particles_with_one_particle = particle2.as_set()
        >>> len(particles_with_one_particle)
        1
        >>> print particles_with_one_particle.x
        [2.0] m
        """
        return ParticlesSubset(self.particles_set, [self.key])

    def as_particle_in_set(self, other):
        return other._get_particle(self.key)

    def get_containing_set(self):
        return self.particles_set._original_set()

    def update(self, attribute_names, callback):
        if self._set_index is None or self._set_version != self.particles_set._get_version():
            object.__setattr__(self, "_set_index", self.particles_set.get_indices_of_keys([self.key])[0])
            object.__setattr__(self, "_set_version", self.particles_set._get_version())
        values = self.particles_set.get_values_in_store((self._set_index), attribute_names)
        updated_values = callback(*values)
        self.particles_set.set_values_in_store((self._set_index), attribute_names, updated_values)
        

def create_particle_set(**args):
    """
    Returns a particle set from the input vector quantities. input should be named
    keyword arguments.
    >>> m=units.kg([ 1.,1.])
    >>> x=units.m([0.,1.])
    >>> particles=create_particle_set(mass=m,x=x)
    >>> print len(particles)
    2
    """
    if len(args)==0:
        raise Exception("provide quantities")

    n=len(list(args.values())[0])

    for a in args:
        nn=len(args[a])
        if nn!=n:
            raise Exception("unequal length quantities")

    particles=Particles(n)

    for a in args:
        setattr(particles,a,args[a])

    return particles




class UpdatingParticlesSubset(AbstractParticleSet):
    """A subset of particles that reruns the selection criteria each time the superset is updated (particles added or removed). 
    Attribute values are not stored by the subset. The subset provides a limited view
    to the particles.

    Particle subset objects are not supposed to be created
    directly. Instead use the ``to_set`` or ``select`` methods.

    """

    def __init__(self, particles, selection_function):
        AbstractParticleSet.__init__(self, particles)
        self._private.particles = particles
        self._private.selection_function = selection_function
        
        self._private.keys = []                     # numpy.array(keys, dtype='uint64')
        self._private.set_of_keys = set([])
        self._private.version = -1
        self._private.indices = None


    def __getitem__(self, index):
        keys = self.get_all_keys_in_store()[index]
        if hasattr(keys, '__iter__'):
            return self._subset(keys)
        else:
            key = keys
            if key > 0 and key < 18446744073709551615: #2**64 - 1
                return self._private.particles._get_particle_unsave(key, self.get_all_indices_in_store()[index])
            else:
                return None

    def _get_version(self):
        return self._private.particles._get_version()



    def compressed(self):
        self._sync_with_set()
        keys = self._private.keys
        return self._subset(keys[numpy.logical_and(keys > 0 ,  keys < 18446744073709551615)])


    def get_valid_particles_mask(self):
        self._sync_with_set()
        keys = self._private.keys
        return numpy.logical_and(keys > 0 ,  keys < 18446744073709551615)


    def unconverted_set(self):
        return UpdatedParticlesSubset(self._private.particles.unconverted_set(), self._private.selection_function)


    def add_particles_to_store(self, keys, attributes = [], values = []):
        """
        Adds particles from to the subset, also
        adds the particles to the superset
        """
        self._private.keys = None
        self._private.set_of_keys = set([])
        self._private.version = -1
        self._private.particles.add_particles_to_store(keys, attributes, values)


    def remove_particles_from_store(self, indices):
        """
        Removes particles from the subset, and removes particles
        from the super set
        """

        indices_to_remove = set(indices)

        index = 0
        index_in_local_list = []
        for x in self._private.indices:
            if x in indices_to_remove:
                index_in_local_list.append(index)
            index += 1
        set_to_remove = set(self._private.keys[index_in_local_list])
        self._private.keys = []     # will rerun the selection
        self._private.set_of_keys = set([])
        self._private.particles.remove_particles_from_store(indices)

        self._private.version = -1
        self._private.indices = None



    def get_values_in_store(self, indices, attributes):
        if indices is None:
            indices = self.get_all_indices_in_store()

        return self._private.particles.get_values_in_store(indices, attributes)


    def set_values_in_store(self, indices, attributes, values):
        if indices is None:
            indices = self.get_all_indices_in_store()

        self._private.particles.set_values_in_store(indices, attributes, values)

    def get_attribute_names_defined_in_store(self):
        return self._private.particles.get_attribute_names_defined_in_store()

    def get_settable_attribute_names_defined_in_store(self):
        return self._private.particles.get_settable_attribute_names_defined_in_store()

    def get_all_keys_in_store(self):
        self._sync_with_set()
            
        return self._private.keys

    def get_all_indices_in_store(self):
        self._sync_with_set()

        return self._private.indices

    def has_key_in_store(self, key):
        self._sync_with_set()
        return key in self._private.set_of_keys


    def _original_set(self):
        return self._private.particles

    def get_timestamp(self):
        return self._original_set().get_timestamp()

    def get_indices_of_keys(self, keys):
        self._sync_with_set()
        
        return self._private.particles.get_indices_of_keys(keys)
        
    def _sync_with_set(self):
        if self._private.particles is None:
            return
        if not self._private.version == self._private.particles._get_version():
            try:
                keys = self._private.particles.get_all_keys_in_store()[self._private.selection_function(self._private.particles)]
                self._private.keys = numpy.array(keys, dtype='uint64')
                self._private.set_of_keys = set(keys)
                self._private.indices = self._private.particles.get_indices_of_keys(self._private.keys)
                self._private.version = self._private.particles._get_version()
            except exceptions.KeysNotInStorageException as ex:
                self._private.indices = ex.found_indices
                self._private.keys = numpy.array(ex.found_keys, dtype='uint64')
                self._private.set_of_keys = set(self._private.keys)
                self._private.version = self._private.particles._get_version()



    def previous_state(self):
        return ParticlesSubset(self._private.particles.previous_state(), self._private.selection_function)


    def difference(self, other):
        self._sync_with_set()
        
        new_set_of_keys = self._private.set_of_keys.difference(other.as_set()._private.set_of_keys)
        return ParticlesSubset(self._private.particles, list(new_set_of_keys))


    def union(self, other):
        """
        Returns a new subset containing the union between
        this set and the provided set.

        >>> particles = Particles(3)
        >>> particles.mass = [10.0, 20.0, 30.0] | units.kg
        >>> subset1 = particles.select(lambda x : x > 25.0 | units.kg, ["mass"])
        >>> subset2 = particles.select(lambda x : x < 15.0 | units.kg, ["mass"])
        >>> union = subset1.union(subset2)
        >>> len(union)
        2
        >>> sorted(union.mass.value_in(units.kg))
        [10.0, 30.0]
        """

        self._sync_with_set()
        new_set_of_keys = self._private.set_of_keys.union(other.as_set()._private.set_of_keys)
        return ParticlesSubset(self._private.particles, list(new_set_of_keys))

    def as_set(self):
        return self

    def copy(self, memento = None, keep_structure = False, filter_attributes = lambda particle_set, x : True):
        self._sync_with_set()
        if keep_structure:
            result = UpdatingParticlesSubset(None, self._private.seleciton_function)
            if memento is None:
                memento = dict()
            memento[id(self)] = result
            if id(self._private.particles) in memento:
                result._private.particles = memento[id(self._private.particles)]
            else:
                result._private.particles = self._private.particles.copy(memento, keep_structure, filter_attributes)
            result._private.selection_function = self._private.seleciton_function
            result._private.keys = []
            result._private.set_of_keys = set([])

            result._private.collection_attributes = self._private.collection_attributes._copy_for_collection(result)

            return result
        else:
            return super(UpdatingParticlesSubset, self).copy(memento, keep_structure, filter_attributes)




