GeezerVison

[submodule "async"]

path = gitdb/ext/async

url = git://github.com/gitpython-developers/async.git

branch = master

[submodule "smmap"]

path = gitdb/ext/smmap

url = git://github.com/Byron/smmap.git

Changelog

0.5.3

* Added support for smmap. SmartMMap allows resources to be managed and controlled. This brings the implementation closer to the way git handles memory maps, such that unused cached memory maps will automatically be freed once a resource limit is hit. The memory limit on 32 bit systems remains though as a sliding mmap implementation is not used for performance reasons.

0.5.2

project = u'GitDB'

copyright = u'2010, Sebastian Thiel'

copyright = u'2011, Sebastian Thiel'

version = '0.5'

release = '0.5.1'

release = '0.5.3'

def _init_externals():

"""Initialize external projects by putting them into the path"""

sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', 'async'))

try:

import async

except ImportError:

raise ImportError("'async' could not be imported, assure it is located in your PYTHONPATH")

#END verify import

for module in ('async', 'smmap'):

sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module))

try:

__import__(module)

except ImportError:

raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module)

#END verify import

#END handel imports

)

from util import (

zlib,

mman,

LazyMixin,

unpack_from,

bin_to_hex,

file_contents_ro_filepath,

)

from fun import (

# Dont use slots as we dynamically bind functions for each version, need a dict for this

# The slots you see here are just to keep track of our instance variables

# __slots__ = ('_indexpath', '_fanout_table', '_data', '_version',

# __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version',

# '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset')

# used in v2 indices

def _set_cache_(self, attr):

if attr == "_packfile_checksum":

self._packfile_checksum = self._data[-40:-20]

self._packfile_checksum = self._cursor.map()[-40:-20]

elif attr == "_packfile_checksum":

self._packfile_checksum = self._data[-20:]

elif attr == "_data":

self._packfile_checksum = self._cursor.map()[-20:]

elif attr == "_cursor":

# Note: We don't lock the file when reading as we cannot be sure

# that we can actually write to the location - it could be a read-only

# alternate for instance

self._data = file_contents_ro_filepath(self._indexpath)

self._cursor = mman.make_cursor(self._indexpath).use_region()

else:

# now its time to initialize everything - if we are here, someone wants

# to access the fanout table or related properties

# CHECK VERSION

self._version = (self._data[:4] == self.index_v2_signature and 2) or 1

mmap = self._cursor.map()

self._version = (mmap[:4] == self.index_v2_signature and 2) or 1

if self._version == 2:

version_id = unpack_from(">L", self._data, 4)[0]

version_id = unpack_from(">L", mmap, 4)[0]

assert version_id == self._version, "Unsupported index version: %i" % version_id

# END assert version

def _entry_v1(self, i):

""":return: tuple(offset, binsha, 0)"""

return unpack_from(">L20s", self._data, 1024 + i*24) + (0, )

return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, )

def _offset_v1(self, i):

"""see ``_offset_v2``"""

return unpack_from(">L", self._data, 1024 + i*24)[0]

return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0]

def _sha_v1(self, i):

"""see ``_sha_v2``"""

base = 1024 + (i*24)+4

return self._data[base:base+20]

return self._cursor.map()[base:base+20]

def _crc_v1(self, i):

"""unsupported"""

def _offset_v2(self, i):

""":return: 32 or 64 byte offset into pack files. 64 byte offsets will only

offset = unpack_from(">L", self._data, self._pack_offset + i * 4)[0]

offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0]

# if the high-bit is set, this indicates that we have to lookup the offset

# in the 64 bit region of the file. The current offset ( lower 31 bits )

# are the index into it

if offset & 0x80000000:

offset = unpack_from(">Q", self._data, self._pack_64_offset + (offset & ~0x80000000) * 8)[0]

offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0]

# END handle 64 bit offset

return offset

def _sha_v2(self, i):

""":return: sha at the given index of this file index instance"""

base = self._sha_list_offset + i * 20

return self._data[base:base+20]

return self._cursor.map()[base:base+20]

def _crc_v2(self, i):

""":return: 4 bytes crc for the object at index i"""

return unpack_from(">L", self._data, self._crc_list_offset + i * 4)[0]

return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0]

#} END access V2

def _read_fanout(self, byte_offset):

"""Generate a fanout table from our data"""

d = self._data

d = self._cursor.map()

out = list()

append = out.append

for i in range(256):

def packfile_checksum(self):

""":return: 20 byte sha representing the sha1 hash of the pack file"""

return self._data[-40:-20]

return self._cursor.map()[-40:-20]

def indexfile_checksum(self):

""":return: 20 byte sha representing the sha1 hash of this index file"""

return self._data[-20:]

return self._cursor.map()[-20:]

def offsets(self):

""":return: sequence of all offsets in the order in which they were written

if self._version == 2:

# read stream to array, convert to tuple

a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears

a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset))

a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset))

# networkbyteorder to something array likes more

if sys.byteorder == 'little':

__slots__ = ('_packpath', '_data', '_size', '_version')

__slots__ = ('_packpath', '_cursor', '_size', '_version')

pack_signature = 0x5041434b # 'PACK'

pack_version_default = 2

self._packpath = packpath

def _set_cache_(self, attr):

if attr == '_data':

self._data = file_contents_ro_filepath(self._packpath)

# read the header information

type_id, self._version, self._size = unpack_from(">LLL", self._data, 0)

# TODO: figure out whether we should better keep the lock, or maybe

# add a .keep file instead ?

else: # must be '_size' or '_version'

# read header info - we do that just with a file stream

type_id, self._version, self._size = unpack(">LLL", open(self._packpath).read(12))

# END handle header

# we fill the whole cache, whichever attribute gets queried first

self._cursor = mman.make_cursor(self._packpath).use_region()

# read the header information

type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0)

# TODO: figure out whether we should better keep the lock, or maybe

# add a .keep file instead ?

if type_id != self.pack_signature:

raise ParseError("Invalid pack signature: %i" % type_id)

#END assert type id

def _iter_objects(self, start_offset, as_stream=True):

"""Handle the actual iteration of objects within this pack"""

data = self._data

data = self._cursor.map()

content_size = len(data) - self.footer_size

cur_offset = start_offset or self.first_object_offset

"""

return self._data

return self._cursor.map()

def checksum(self):

""":return: 20 byte sha1 hash on all object sha's contained in this file"""

return self._data[-20:]

return self._cursor.map()[-20:]

def path(self):

""":return: path to the packfile"""

out = list()

data = self._cursor.map()

while True:

ostream = pack_object_at(self._data, offset, True)[1]

ostream = pack_object_at(data, offset, True)[1]

out.append(ostream)

if ostream.type_id == OFS_DELTA:

offset = ostream.pack_offset - ostream.delta_info

return pack_object_at(self._data, offset or self.first_object_offset, False)[1]

return pack_object_at(self._cursor.map(), offset or self.first_object_offset, False)[1]

def stream(self, offset):

"""Retrieve an object at the given file-relative offset as stream along with its information

return pack_object_at(self._data, offset or self.first_object_offset, True)[1]

return pack_object_at(self._cursor.map(), offset or self.first_object_offset, True)[1]

def stream_iter(self, start_offset=0):

"""

sha = self._index.sha(index)

# END assure sha is present ( in output )

offset = self._index.offset(index)

type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset))

type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._cursor.map(), offset))

if as_stream:

if type_id not in delta_types:

packstream = self._pack.stream(offset)

from time import time

import random

from nose import SkipTest

class TestPackedDBPerformance(TestBigRepoR):

def _test_pack_random_access(self):

def test_pack_random_access(self):

pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack"))

# sha lookup

print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed)

def test_correctness(self):

raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly")

pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack"))

# disabled for now as it used to work perfectly, checking big repositories takes a long time

print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)"

from async import ThreadPool

from smmap import (

StaticWindowMapManager,

SlidingWindowMapBuffer

)

mman = StaticWindowMapManager()

try:

import hashlib

close(fd)

# END assure file is closed

def sliding_ro_buffer(filepath, flags=0):

""":return: a buffer compatible object which uses our mapped memory manager internally

return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags)

def to_hex_sha(sha):

""":return: hexified version of sha"""

if len(sha) == 40:

setup(cmdclass={'build_ext':build_ext_nofail},

name = "gitdb",

version = "0.5.2",

version = "0.5.3",

description = "Git Object Database",

author = "Sebastian Thiel",

author_email = "[email protected]",

ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])],

license = "BSD License",

zip_safe=False,

requires=('async (>=0.6.1)',),

install_requires='async >= 0.6.1',

requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'),

install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'),

long_description = """GitDB is a pure-Python git object database"""

)