To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to object_store.py
- browse files at revision 0.257.2
- compare with another revision
- download diff
- download tarball
- view history from revision 0.257.2
- Committer: Jelmer Vernooij
- Date: 2010-06-28 23:48:36 UTC
- mto: (0.200.953 trunk)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100628234836-96qxjh7g2xxedblp
Fix transportgit.
- files added:
- notes/git-serve.txt
- files renamed:
- shamap.py => cache.py
- tests/test_shamap.py => tests/test_cache.py
- files modified:
- Makefile
added
removed
object_store.py
18
18
19
19
from dulwich.objects import (
20
20
Blob,
21
Tree,
21
22
sha_to_hex,
22
23
)
23
24
from dulwich.object_store import (
25
26
)
26
27
27
28
from bzrlib import (
28
debug,
29
29
errors,
30
lru_cache,
30
31
trace,
31
32
ui,
33
urlutils,
32
34
)
33
35
from bzrlib.revision import (
34
36
NULL_REVISION,
39
41
directory_to_tree,
40
42
extract_unusual_modes,
41
43
mapping_registry,
42
)
43
from bzrlib.plugins.git.shamap import (
44
SqliteGitShaMap,
45
TdbGitShaMap,
46
)
44
symlink_to_blob,
45
)
46
from bzrlib.plugins.git.cache import (
47
from_repository as cache_from_repository,
48
)
49
50
import posixpath
51
import stat
47
52
48
53
49
54
def get_object_store(repo, mapping=None):
53
58
return BazaarObjectStore(repo, mapping)
54
59
55
60
61
MAX_TREE_CACHE_SIZE = 50 * 1024 * 1024
62
63
64
class LRUTreeCache(object):
65
66
def __init__(self, repository):
67
def approx_tree_size(tree):
68
# Very rough estimate, 1k per inventory entry
69
return len(tree.inventory) * 1024
70
self.repository = repository
71
self._cache = lru_cache.LRUSizeCache(max_size=MAX_TREE_CACHE_SIZE,
72
after_cleanup_size=None, compute_size=approx_tree_size)
73
74
def revision_tree(self, revid):
75
try:
76
return self._cache[revid]
77
except KeyError:
78
tree = self.repository.revision_tree(revid)
79
self.add(tree)
80
return tree
81
82
def iter_revision_trees(self, revids):
83
trees = dict([(k, self._cache.get(k)) for k in revids])
84
for tree in self.repository.revision_trees(
85
[r for r, v in trees.iteritems() if v is None]):
86
trees[tree.get_revision_id()] = tree
87
self.add(tree)
88
return (trees[r] for r in revids)
89
90
def revision_trees(self, revids):
91
return list(self.iter_revision_trees(revids))
92
93
def add(self, tree):
94
self._cache.add(tree.get_revision_id(), tree)
95
96
97
def _find_missing_bzr_revids(get_parent_map, want, have):
98
"""Find the revisions that have to be pushed.
99
100
:param get_parent_map: Function that returns the parents for a sequence
101
of revisions.
102
:param want: Revisions the target wants
103
:param have: Revisions the target already has
104
:return: Set of revisions to fetch
105
"""
106
pending = want - have
107
processed = set()
108
todo = set()
109
while pending:
110
processed.update(pending)
111
next_map = get_parent_map(pending)
112
next_pending = set()
113
for item in next_map.iteritems():
114
if item[0] in have:
115
continue
116
todo.add(item[0])
117
next_pending.update(p for p in item[1] if p not in processed)
118
pending = next_pending
119
if NULL_REVISION in todo:
120
todo.remove(NULL_REVISION)
121
return todo
122
123
124
def _check_expected_sha(expected_sha, object):
125
"""Check whether an object matches an expected SHA.
126
127
:param expected_sha: None or expected SHA as either binary or as hex digest
128
:param object: Object to verify
129
"""
130
if expected_sha is None:
131
return
132
if len(expected_sha) == 40:
133
if expected_sha != object.sha().hexdigest():
134
raise AssertionError("Invalid sha for %r: %s" % (object,
135
expected_sha))
136
elif len(expected_sha) == 20:
137
if expected_sha != object.sha().digest():
138
raise AssertionError("Invalid sha for %r: %s" % (object,
139
sha_to_hex(expected_sha)))
140
else:
141
raise AssertionError("Unknown length %d for %r" % (len(expected_sha),
142
expected_sha))
143
144
145
def _tree_to_objects(tree, parent_trees, idmap, unusual_modes,
146
dummy_file_name=None):
147
"""Iterate over the objects that were introduced in a revision.
148
149
:param idmap: id map
150
:param parent_trees: Parent revision trees
151
:param unusual_modes: Unusual file modes dictionary
152
:param dummy_file_name: File name to use for dummy files
153
in empty directories. None to skip empty directories
154
:return: Yields (path, object, ie) entries
155
"""
156
new_trees = {}
157
new_blobs = []
158
shamap = {}
159
try:
160
base_tree = parent_trees[0]
161
other_parent_trees = parent_trees[1:]
162
except IndexError:
163
base_tree = tree._repository.revision_tree(NULL_REVISION)
164
other_parent_trees = []
165
def find_unchanged_parent_ie(ie, parent_trees):
166
assert ie.kind in ("symlink", "file")
167
for ptree in parent_trees:
168
try:
169
pie = ptree.inventory[ie.file_id]
170
except errors.NoSuchId:
171
pass
172
else:
173
if (pie.text_sha1 == ie.text_sha1 and
174
pie.kind == ie.kind and
175
pie.symlink_target == ie.symlink_target):
176
return pie
177
raise KeyError
178
179
# Find all the changed blobs
180
for (file_id, path, changed_content, versioned, parent, name, kind,
181
executable) in tree.iter_changes(base_tree):
182
if kind[1] == "file":
183
ie = tree.inventory[file_id]
184
if changed_content:
185
try:
186
pie = find_unchanged_parent_ie(ie, other_parent_trees)
187
except KeyError:
188
pass
189
else:
190
try:
191
shamap[ie.file_id] = idmap.lookup_blob_id(
192
pie.file_id, pie.revision)
193
except KeyError:
194
# no-change merge ?
195
blob = Blob()
196
blob.data = tree.get_file_text(ie.file_id)
197
shamap[ie.file_id] = blob.id
198
if not file_id in shamap:
199
new_blobs.append((path[1], ie))
200
new_trees[posixpath.dirname(path[1])] = parent[1]
201
elif kind[1] == "symlink":
202
ie = tree.inventory[file_id]
203
if changed_content:
204
blob = symlink_to_blob(ie)
205
shamap[file_id] = blob.id
206
try:
207
find_unchanged_parent_ie(ie, other_parent_trees)
208
except KeyError:
209
yield path[1], blob, ie
210
new_trees[posixpath.dirname(path[1])] = parent[1]
211
elif kind[1] not in (None, "directory"):
212
raise AssertionError(kind[1])
213
if path[0] is not None and parent[0] in tree.inventory and tree.inventory[parent[0]].kind == "directory":
214
# Removal
215
new_trees[posixpath.dirname(path[0])] = parent[0]
216
217
# Fetch contents of the blobs that were changed
218
for (path, ie), chunks in tree.iter_files_bytes(
219
[(ie.file_id, (path, ie)) for (path, ie) in new_blobs]):
220
obj = Blob()
221
obj.chunked = chunks
222
yield path, obj, ie
223
shamap[ie.file_id] = obj.id
224
225
for path in unusual_modes:
226
parent_path = posixpath.dirname(path)
227
new_trees[parent_path] = tree.path2id(parent_path)
228
229
trees = {}
230
while new_trees:
231
items = new_trees.items()
232
new_trees = {}
233
for path, file_id in items:
234
parent_id = tree.inventory[file_id].parent_id
235
if parent_id is not None:
236
parent_path = urlutils.dirname(path)
237
new_trees[parent_path] = parent_id
238
trees[path] = file_id
239
240
def ie_to_hexsha(ie):
241
try:
242
return shamap[ie.file_id]
243
except KeyError:
244
# FIXME: Should be the same as in parent
245
if ie.kind in ("file", "symlink"):
246
try:
247
return idmap.lookup_blob_id(ie.file_id, ie.revision)
248
except KeyError:
249
# no-change merge ?
250
blob = Blob()
251
blob.data = tree.get_file_text(ie.file_id)
252
return blob.id
253
elif ie.kind == "directory":
254
# Not all cache backends store the tree information,
255
# calculate again from scratch
256
ret = directory_to_tree(ie, ie_to_hexsha, unusual_modes,
257
dummy_file_name)
258
if ret is None:
259
return ret
260
return ret.id
261
else:
262
raise AssertionError
263
264
for path in sorted(trees.keys(), reverse=True):
265
ie = tree.inventory[trees[path]]
266
assert ie.kind == "directory"
267
obj = directory_to_tree(ie, ie_to_hexsha, unusual_modes,
268
dummy_file_name)
269
if obj is not None:
270
yield path, obj, ie
271
shamap[ie.file_id] = obj.id
272
273
56
274
class BazaarObjectStore(BaseObjectStore):
57
275
"""A Git-style object store backed onto a Bazaar repository."""
58
276
62
280
self.mapping = default_mapping
63
281
else:
64
282
self.mapping = mapping
65
self._idmap = SqliteGitShaMap.from_repository(repository)
283
self._cache = cache_from_repository(repository)
284
self._content_cache_types = ("tree")
285
self.start_write_group = self._cache.idmap.start_write_group
286
self.abort_write_group = self._cache.idmap.abort_write_group
287
self.commit_write_group = self._cache.idmap.commit_write_group
288
self.tree_cache = LRUTreeCache(self.repository)
66
289
67
290
def _update_sha_map(self, stop_revision=None):
68
291
graph = self.repository.get_graph()
70
293
heads = graph.heads(self.repository.all_revision_ids())
71
294
else:
72
295
heads = set([stop_revision])
73
missing_revids = self._idmap.missing_revisions(heads)
296
missing_revids = self._cache.idmap.missing_revisions(heads)
74
297
while heads:
75
298
parents = graph.get_parent_map(heads)
76
299
todo = set()
77
300
for p in parents.values():
78
301
todo.update([x for x in p if x not in missing_revids])
79
heads = self._idmap.missing_revisions(todo)
302
heads = self._cache.idmap.missing_revisions(todo)
80
303
missing_revids.update(heads)
81
304
if NULL_REVISION in missing_revids:
82
305
missing_revids.remove(NULL_REVISION)
83
pb = ui.ui_factory.nested_progress_bar()
306
missing_revids = self.repository.has_revisions(missing_revids)
307
if not missing_revids:
308
return
309
self.start_write_group()
84
310
try:
85
for i, revid in enumerate(graph.iter_topo_order(missing_revids)):
86
pb.update("updating git map", i, len(missing_revids))
87
self._update_sha_map_revision(revid)
88
finally:
89
pb.finished()
90
self._idmap.commit_write_group()
311
pb = ui.ui_factory.nested_progress_bar()
312
try:
313
for i, revid in enumerate(graph.iter_topo_order(missing_revids)):
314
trace.mutter('processing %r', revid)
315
pb.update("updating git map", i, len(missing_revids))
316
self._update_sha_map_revision(revid)
317
finally:
318
pb.finished()
319
except:
320
self.abort_write_group()
321
raise
322
else:
323
self.commit_write_group()
91
324
92
325
def __iter__(self):
93
326
self._update_sha_map()
94
return iter(self._idmap.sha1s())
327
return iter(self._cache.idmap.sha1s())
95
328
96
def _revision_to_commit(self, rev, tree_sha):
329
def _reconstruct_commit(self, rev, tree_sha, roundtrip):
97
330
def parent_lookup(revid):
98
331
try:
99
332
return self._lookup_revision_sha1(revid)
100
333
except errors.NoSuchRevision:
101
trace.warning("Ignoring ghost parent %s", revid)
102
334
return None
103
return self.mapping.export_commit(rev, tree_sha, parent_lookup)
104
105
def _update_sha_map_revision(self, revid):
106
inv = self.repository.get_inventory(revid)
107
rev = self.repository.get_revision(revid)
335
return self.mapping.export_commit(rev, tree_sha, parent_lookup,
336
roundtrip)
337
338
def _create_fileid_map_blob(self, inv):
339
# FIXME: This can probably be a lot more efficient,
340
# not all files necessarily have to be processed.
341
file_ids = {}
342
for (path, ie) in inv.iter_entries():
343
if self.mapping.generate_file_id(path) != ie.file_id:
344
file_ids[path] = ie.file_id
345
return self.mapping.export_fileid_map(file_ids)
346
347
def _revision_to_objects(self, rev, tree, roundtrip):
348
"""Convert a revision to a set of git objects.
349
350
:param rev: Bazaar revision object
351
:param tree: Bazaar revision tree
352
:param roundtrip: Whether to roundtrip all Bazaar revision data
353
"""
108
354
unusual_modes = extract_unusual_modes(rev)
109
tree_sha = self._get_ie_sha1(inv.root, inv, unusual_modes)
110
commit_obj = self._revision_to_commit(rev, tree_sha)
355
present_parents = self.repository.has_revisions(rev.parent_ids)
356
parent_trees = self.tree_cache.revision_trees(
357
[p for p in rev.parent_ids if p in present_parents])
358
root_tree = None
359
for path, obj, ie in _tree_to_objects(tree, parent_trees,
360
self._cache.idmap, unusual_modes, self.mapping.BZR_DUMMY_FILE):
361
if path == "":
362
root_tree = obj
363
root_ie = ie
364
# Don't yield just yet
365
else:
366
yield path, obj, ie
367
if root_tree is None:
368
# Pointless commit - get the tree sha elsewhere
369
if not rev.parent_ids:
370
root_tree = Tree()
371
else:
372
base_sha1 = self._lookup_revision_sha1(rev.parent_ids[0])
373
root_tree = self[self[base_sha1].tree]
374
root_ie = tree.inventory.root
375
if roundtrip and self.mapping.BZR_FILE_IDS_FILE is not None:
376
b = self._create_fileid_map_blob(tree.inventory)
377
if b is not None:
378
root_tree[self.mapping.BZR_FILE_IDS_FILE] = ((stat.S_IFREG | 0644), b.id)
379
yield self.mapping.BZR_FILE_IDS_FILE, b, None
380
yield "", root_tree, root_ie
381
commit_obj = self._reconstruct_commit(rev, root_tree.id,
382
roundtrip=roundtrip)
111
383
try:
112
foreign_revid, mapping = mapping_registry.parse_revision_id(revid)
384
foreign_revid, mapping = mapping_registry.parse_revision_id(
385
rev.revision_id)
113
386
except errors.InvalidRevisionId:
114
387
pass
115
388
else:
116
if foreign_revid != commit_obj.id:
117
if not "fix-shamap" in debug.debug_flags:
118
raise AssertionError("recreated git commit had different sha1: expected %s, got %s" % (foreign_revid, commit_obj.id))
119
self._idmap.add_entry(commit_obj.id, "commit", (revid, tree_sha))
120
121
def _check_expected_sha(self, expected_sha, object):
122
if expected_sha is None:
123
return
124
if len(expected_sha) == 40:
125
if expected_sha != object.sha().hexdigest():
126
raise AssertionError("Invalid sha for %r: %s" % (object, expected_sha))
127
elif len(expected_sha) == 20:
128
if expected_sha != object.sha().digest():
129
raise AssertionError("Invalid sha for %r: %s" % (object, sha_to_hex(expected_sha)))
130
else:
131
raise AssertionError("Unknown length %d for %r" % (len(expected_sha), expected_sha))
132
133
def _get_ie_object(self, entry, inv, unusual_modes):
134
if entry.kind == "directory":
135
return self._get_tree(entry.file_id, inv.revision_id, inv, unusual_modes)
136
elif entry.kind == "symlink":
137
return self._get_blob_for_symlink(entry.symlink_target)
138
elif entry.kind == "file":
139
return self._get_blob_for_file(entry.file_id, entry.revision)
140
else:
141
raise AssertionError("unknown entry kind '%s'" % entry.kind)
142
143
def _get_ie_object_or_sha1(self, entry, inv, unusual_modes):
144
if entry.kind == "directory":
145
try:
146
return self._idmap.lookup_tree(entry.file_id, inv.revision_id), None
147
except KeyError:
148
ret = self._get_ie_object(entry, inv, unusual_modes)
149
if ret is None:
150
hexsha = None
151
else:
152
hexsha = ret.id
153
self._idmap.add_entry(hexsha, "tree", (entry.file_id, inv.revision_id))
154
return hexsha, ret
155
elif entry.kind in ("file", "symlink"):
156
try:
157
return self._idmap.lookup_blob(entry.file_id, entry.revision), None
158
except KeyError:
159
ret = self._get_ie_object(entry, inv, unusual_modes)
160
self._idmap.add_entry(ret.id, "blob", (entry.file_id, entry.revision))
161
return ret.id, ret
162
else:
163
raise AssertionError("unknown entry kind '%s'" % entry.kind)
164
165
def _get_ie_sha1(self, entry, inv, unusual_modes):
166
return self._get_ie_object_or_sha1(entry, inv, unusual_modes)[0]
167
168
def _get_blob_for_symlink(self, symlink_target, expected_sha=None):
169
"""Return a Git Blob object for symlink.
170
171
:param symlink_target: target of symlink.
172
"""
173
if type(symlink_target) == unicode:
174
symlink_target = symlink_target.encode('utf-8')
175
blob = Blob()
176
blob._text = symlink_target
177
self._check_expected_sha(expected_sha, blob)
178
return blob
179
180
def _get_blob_for_file(self, fileid, revision, expected_sha=None):
181
"""Return a Git Blob object from a fileid and revision stored in bzr.
182
183
:param fileid: File id of the text
184
:param revision: Revision of the text
185
"""
186
blob = Blob()
187
chunks = self.repository.iter_files_bytes([(fileid, revision, None)]).next()[1]
188
blob._text = "".join(chunks)
189
self._check_expected_sha(expected_sha, blob)
190
return blob
191
192
def _get_blob(self, fileid, revision, expected_sha=None):
193
"""Return a Git Blob object from a fileid and revision stored in bzr.
194
195
:param fileid: File id of the text
196
:param revision: Revision of the text
197
"""
198
inv = self.repository.get_inventory(revision)
199
entry = inv[fileid]
200
201
if entry.kind == 'file':
202
return self._get_blob_for_file(entry.file_id, entry.revision,
203
expected_sha=expected_sha)
204
elif entry.kind == 'symlink':
205
return self._get_blob_for_symlink(entry.symlink_target,
206
expected_sha=expected_sha)
207
else:
208
raise AssertionError
209
210
def _get_tree(self, fileid, revid, inv, unusual_modes, expected_sha=None):
389
_check_expected_sha(foreign_revid, commit_obj)
390
yield None, commit_obj, None
391
392
def _get_updater(self, rev):
393
return self._cache.get_updater(rev)
394
395
def _update_sha_map_revision(self, revid):
396
rev = self.repository.get_revision(revid)
397
tree = self.tree_cache.revision_tree(rev.revision_id)
398
updater = self._get_updater(rev)
399
for path, obj, ie in self._revision_to_objects(rev, tree,
400
roundtrip=True):
401
updater.add_object(obj, ie, path)
402
commit_obj = updater.finish()
403
return commit_obj.id
404
405
def _reconstruct_blobs(self, keys):
406
"""Return a Git Blob object from a fileid and revision stored in bzr.
407
408
:param fileid: File id of the text
409
:param revision: Revision of the text
410
"""
411
stream = self.repository.iter_files_bytes(
412
((key[0], key[1], key) for key in keys))
413
for (fileid, revision, expected_sha), chunks in stream:
414
blob = Blob()
415
blob.chunked = chunks
416
if blob.id != expected_sha and blob.data == "":
417
# Perhaps it's a symlink ?
418
tree = self.tree_cache.revision_tree(revision)
419
entry = tree.inventory[fileid]
420
if entry.kind == 'symlink':
421
blob = symlink_to_blob(entry)
422
_check_expected_sha(expected_sha, blob)
423
yield blob
424
425
def _reconstruct_tree(self, fileid, revid, inv, unusual_modes,
426
expected_sha=None):
211
427
"""Return a Git Tree object from a file id and a revision stored in bzr.
212
428
213
429
:param fileid: fileid in the tree.
214
430
:param revision: Revision of the tree.
215
431
"""
216
tree = directory_to_tree(inv[fileid],
217
lambda ie: self._get_ie_sha1(ie, inv, unusual_modes),
218
unusual_modes)
219
self._check_expected_sha(expected_sha, tree)
432
def get_ie_sha1(entry):
433
if entry.kind == "directory":
434
try:
435
return self._cache.idmap.lookup_tree_id(entry.file_id,
436
revid)
437
except (NotImplementedError, KeyError):
438
obj = self._reconstruct_tree(entry.file_id, revid, inv,
439
unusual_modes)
440
if obj is None:
441
return None
442
else:
443
return obj.id
444
elif entry.kind in ("file", "symlink"):
445
try:
446
return self._cache.idmap.lookup_blob_id(entry.file_id,
447
entry.revision)
448
except KeyError:
449
# no-change merge?
450
return self._reconstruct_blobs(
451
[(entry.file_id, entry.revision, None)]).next().id
452
else:
453
raise AssertionError("unknown entry kind '%s'" % entry.kind)
454
tree = directory_to_tree(inv[fileid], get_ie_sha1, unusual_modes,
455
self.mapping.BZR_DUMMY_FILE)
456
if (inv.root.file_id == fileid and
457
self.mapping.BZR_FILE_IDS_FILE is not None):
458
b = self._create_fileid_map_blob(inv)
459
# If this is the root tree, add the file ids
460
tree[self.mapping.BZR_FILE_IDS_FILE] = ((stat.S_IFREG | 0644), b.id)
461
_check_expected_sha(expected_sha, tree)
220
462
return tree
221
463
222
def _get_commit(self, rev, tree_sha, expected_sha=None):
223
commit = self._revision_to_commit(rev, tree_sha)
224
self._check_expected_sha(expected_sha, commit)
225
return commit
226
227
464
def get_parents(self, sha):
228
465
"""Retrieve the parents of a Git commit by SHA1.
229
466
234
471
235
472
def _lookup_revision_sha1(self, revid):
236
473
"""Return the SHA1 matching a Bazaar revision."""
474
from dulwich.protocol import ZERO_SHA
237
475
if revid == NULL_REVISION:
238
return "0" * 40
476
return ZERO_SHA
239
477
try:
240
return self._idmap.lookup_commit(revid)
478
return self._cache.idmap.lookup_commit(revid)
241
479
except KeyError:
242
480
try:
243
481
return mapping_registry.parse_revision_id(revid)[0]
244
482
except errors.InvalidRevisionId:
245
self._update_sha_map(revid)
246
return self._idmap.lookup_commit(revid)
483
self.repository.lock_read()
484
try:
485
self._update_sha_map(revid)
486
finally:
487
self.repository.unlock()
488
return self._cache.idmap.lookup_commit(revid)
247
489
248
490
def get_raw(self, sha):
249
491
"""Get the raw representation of a Git object by SHA1.
256
498
def __contains__(self, sha):
257
499
# See if sha is in map
258
500
try:
259
(type, type_data) = self._lookup_git_sha(sha)
501
(type, type_data) = self.lookup_git_sha(sha)
260
502
if type == "commit":
261
503
return self.repository.has_revision(type_data[0])
262
504
elif type == "blob":
267
509
raise AssertionError("Unknown object type '%s'" % type)
268
510
except KeyError:
269
511
return False
270
else:
271
return True
272
273
def _lookup_git_sha(self, sha):
274
# See if sha is in map
275
try:
276
return self._idmap.lookup_git_sha(sha)
277
except KeyError:
278
# if not, see if there are any unconverted revisions and add them
279
# to the map, search for sha in map again
280
self._update_sha_map()
281
return self._idmap.lookup_git_sha(sha)
512
513
def lookup_git_shas(self, shas, update_map=True):
514
ret = {}
515
for sha in shas:
516
try:
517
ret[sha] = self._cache.idmap.lookup_git_sha(sha)
518
except KeyError:
519
if update_map:
520
# if not, see if there are any unconverted revisions and add
521
# them to the map, search for sha in map again
522
self._update_sha_map()
523
update_map = False
524
try:
525
ret[sha] = self._cache.idmap.lookup_git_sha(sha)
526
except KeyError:
527
pass
528
return ret
529
530
def lookup_git_sha(self, sha, update_map=True):
531
return self.lookup_git_shas([sha], update_map=update_map)[sha]
282
532
283
533
def __getitem__(self, sha):
284
(type, type_data) = self._lookup_git_sha(sha)
534
if self._cache.content_cache is not None:
535
try:
536
return self._cache.content_cache[sha]
537
except KeyError:
538
pass
539
(type, type_data) = self.lookup_git_sha(sha)
285
540
# convert object to git object
286
541
if type == "commit":
542
(revid, tree_sha) = type_data
287
543
try:
288
rev = self.repository.get_revision(type_data[0])
544
rev = self.repository.get_revision(revid)
289
545
except errors.NoSuchRevision:
290
trace.mutter('entry for %s %s in shamap: %r, but not found in repository', type, sha, type_data)
546
trace.mutter('entry for %s %s in shamap: %r, but not found in '
547
'repository', type, sha, type_data)
291
548
raise KeyError(sha)
292
return self._get_commit(rev, type_data[1], expected_sha=sha)
549
commit = self._reconstruct_commit(rev, tree_sha, roundtrip=True)
550
_check_expected_sha(sha, commit)
551
return commit
293
552
elif type == "blob":
294
return self._get_blob(type_data[0], type_data[1], expected_sha=sha)
553
(fileid, revision) = type_data
554
return self._reconstruct_blobs([(fileid, revision, sha)]).next()
295
555
elif type == "tree":
556
(fileid, revid) = type_data
296
557
try:
297
inv = self.repository.get_inventory(type_data[1])
298
rev = self.repository.get_revision(type_data[1])
558
tree = self.tree_cache.revision_tree(revid)
559
rev = self.repository.get_revision(revid)
299
560
except errors.NoSuchRevision:
300
561
trace.mutter('entry for %s %s in shamap: %r, but not found in repository', type, sha, type_data)
301
562
raise KeyError(sha)
302
563
unusual_modes = extract_unusual_modes(rev)
303
564
try:
304
return self._get_tree(type_data[0], type_data[1], inv, unusual_modes,
305
expected_sha=sha)
565
return self._reconstruct_tree(fileid, revid, tree.inventory,
566
unusual_modes, expected_sha=sha)
306
567
except errors.NoSuchRevision:
307
568
raise KeyError(sha)
308
569
else:
309
570
raise AssertionError("Unknown object type '%s'" % type)
571
572
def generate_lossy_pack_contents(self, have, want, progress=None,
573
get_tagged=None):
574
return self.generate_pack_contents(have, want, progress, get_tagged,
575
lossy=True)
576
577
def generate_pack_contents(self, have, want, progress=None,
578
get_tagged=None, lossy=False):
579
"""Iterate over the contents of a pack file.
580
581
:param have: List of SHA1s of objects that should not be sent
582
:param want: List of SHA1s of objects that should be sent
583
"""
584
processed = set()
585
ret = self.lookup_git_shas(have + want)
586
for commit_sha in have:
587
try:
588
(type, (revid, tree_sha)) = ret[commit_sha]
589
except KeyError:
590
pass
591
else:
592
assert type == "commit"
593
processed.add(revid)
594
pending = set()
595
for commit_sha in want:
596
if commit_sha in have:
597
continue
598
try:
599
(type, (revid, tree_sha)) = ret[commit_sha]
600
except KeyError:
601
pass
602
else:
603
assert type == "commit"
604
pending.add(revid)
605
606
todo = _find_missing_bzr_revids(self.repository.get_parent_map,
607
pending, processed)
608
trace.mutter('sending revisions %r', todo)
609
ret = []
610
pb = ui.ui_factory.nested_progress_bar()
611
try:
612
for i, revid in enumerate(todo):
613
pb.update("generating git objects", i, len(todo))
614
rev = self.repository.get_revision(revid)
615
tree = self.tree_cache.revision_tree(revid)
616
for path, obj, ie in self._revision_to_objects(rev, tree,
617
roundtrip=not lossy):
618
ret.append((obj, path))
619
finally:
620
pb.finished()
621
return ret
622
623
def add_thin_pack(self):
624
import tempfile
625
import os
626
fd, path = tempfile.mkstemp(suffix=".pack")
627
f = os.fdopen(fd, 'wb')
628
def commit():
629
from dulwich.pack import PackData, Pack
630
from bzrlib.plugins.git.fetch import import_git_objects
631
os.fsync(fd)
632
f.close()
633
if os.path.getsize(path) == 0:
634
return
635
pd = PackData(path)
636
pd.create_index_v2(path[:-5]+".idx", self.object_store.get_raw)
637
638
p = Pack(path[:-5])
639
self.repository.lock_write()
640
try:
641
self.repository.start_write_group()
642
try:
643
import_git_objects(self.repository, self.mapping,
644
p.iterobjects(get_raw=self.get_raw),
645
self.object_store)
646
except:
647
self.repository.abort_write_group()
648
raise
649
else:
650
self.repository.commit_write_group()
651
finally:
652
self.repository.unlock()
653
return f, commit
654
655
# The pack isn't kept around anyway, so no point
656
# in treating full packs different from thin packs
657
add_pack = add_thin_pack
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0