53
50
from bzrlib.testament import Testament
56
from bzrlib.decorators import needs_read_lock, needs_write_lock, only_raises
53
from bzrlib.decorators import needs_read_lock, needs_write_lock
57
54
from bzrlib.inter import InterObject
58
55
from bzrlib.inventory import (
64
from bzrlib.lock import _RelockDebugMixin
65
61
from bzrlib import registry
66
62
from bzrlib.trace import (
67
63
log_exception_quietly, note, mutter, mutter_callsite, warning)
210
206
# an inventory delta was accumulated without creating a new
212
208
basis_id = self.basis_delta_revision
213
# We ignore the 'inventory' returned by add_inventory_by_delta
214
# because self.new_inventory is used to hint to the rest of the
215
# system what code path was taken
216
self.inv_sha1, _ = self.repository.add_inventory_by_delta(
209
self.inv_sha1 = self.repository.add_inventory_by_delta(
217
210
basis_id, self._basis_delta, self._new_revision_id,
860
853
# versioned roots do not change unless the tree found a change.
863
class RepositoryWriteLockResult(object):
864
"""The result of write locking a repository.
866
:ivar repository_token: The token obtained from the underlying lock, or
868
:ivar unlock: A callable which will unlock the lock.
871
def __init__(self, unlock, repository_token):
872
self.repository_token = repository_token
876
856
######################################################################
880
class Repository(_RelockDebugMixin, bzrdir.ControlComponent):
860
class Repository(object):
881
861
"""Repository holding history for one or more branches.
883
863
The repository holds and retrieves historical information including
1042
1022
:seealso: add_inventory, for the contract.
1044
inv_lines = self._serializer.write_inventory_to_lines(inv)
1024
inv_lines = self._serialise_inventory_to_lines(inv)
1045
1025
return self._inventory_add_lines(revision_id, parents,
1046
1026
inv_lines, check_content=False)
1254
1234
"""Check a single text from this repository."""
1255
1235
if kind == 'inventories':
1256
1236
rev_id = record.key[0]
1257
inv = self._deserialise_inventory(rev_id,
1237
inv = self.deserialise_inventory(rev_id,
1258
1238
record.get_bytes_as('fulltext'))
1259
1239
if last_object is not None:
1260
1240
delta = inv._make_delta(last_object)
1305
1285
:param _format: The format of the repository on disk.
1306
1286
:param a_bzrdir: The BzrDir of the repository.
1288
In the future we will have a single api for all stores for
1289
getting file texts, inventories and revisions, then
1290
this construct will accept instances of those things.
1308
# In the future we will have a single api for all stores for
1309
# getting file texts, inventories and revisions, then
1310
# this construct will accept instances of those things.
1311
1292
super(Repository, self).__init__()
1312
1293
self._format = _format
1313
1294
# the following are part of the public API for Repository:
1319
1300
self._reconcile_does_inventory_gc = True
1320
1301
self._reconcile_fixes_text_parents = False
1321
1302
self._reconcile_backsup_inventory = True
1303
# not right yet - should be more semantically clear ?
1305
# TODO: make sure to construct the right store classes, etc, depending
1306
# on whether escaping is required.
1307
self._warn_if_deprecated()
1322
1308
self._write_group = None
1323
1309
# Additional places to query for data.
1324
1310
self._fallback_repositories = []
1325
1311
# An InventoryEntry cache, used during deserialization
1326
1312
self._inventory_entry_cache = fifo_cache.FIFOCache(10*1024)
1327
# Is it safe to return inventory entries directly from the entry cache,
1328
# rather copying them?
1329
self._safe_to_return_from_cache = False
1332
def user_transport(self):
1333
return self.bzrdir.user_transport
1336
def control_transport(self):
1337
return self._transport
1339
1314
def __repr__(self):
1340
1315
if self._fallback_repositories:
1389
1364
data during reads, and allows a 'write_group' to be obtained. Write
1390
1365
groups must be used for actual data insertion.
1392
A token should be passed in if you know that you have locked the object
1393
some other way, and need to synchronise this object's state with that
1396
XXX: this docstring is duplicated in many places, e.g. lockable_files.py
1398
1367
:param token: if this is already locked, then lock_write will fail
1399
1368
unless the token matches the existing lock.
1400
1369
:returns: a token if this instance supports tokens, otherwise None.
1403
1372
:raises MismatchedToken: if the specified token doesn't match the token
1404
1373
of the existing lock.
1405
1374
:seealso: start_write_group.
1406
:return: A RepositoryWriteLockResult.
1376
A token should be passed in if you know that you have locked the object
1377
some other way, and need to synchronise this object's state with that
1380
XXX: this docstring is duplicated in many places, e.g. lockable_files.py
1408
1382
locked = self.is_locked()
1409
token = self.control_files.lock_write(token=token)
1383
result = self.control_files.lock_write(token=token)
1411
self._warn_if_deprecated()
1412
self._note_lock('w')
1413
1385
for repo in self._fallback_repositories:
1414
1386
# Writes don't affect fallback repos
1415
1387
repo.lock_read()
1416
1388
self._refresh_data()
1417
return RepositoryWriteLockResult(self.unlock, token)
1419
1391
def lock_read(self):
1420
"""Lock the repository for read operations.
1422
:return: An object with an unlock method which will release the lock
1425
1392
locked = self.is_locked()
1426
1393
self.control_files.lock_read()
1428
self._warn_if_deprecated()
1429
self._note_lock('r')
1430
1395
for repo in self._fallback_repositories:
1431
1396
repo.lock_read()
1432
1397
self._refresh_data()
1435
1399
def get_physical_lock_status(self):
1436
1400
return self.control_files.get_physical_lock_status()
1497
1461
# now gather global repository information
1498
1462
# XXX: This is available for many repos regardless of listability.
1499
if self.user_transport.listable():
1463
if self.bzrdir.root_transport.listable():
1500
1464
# XXX: do we want to __define len__() ?
1501
1465
# Maybe the versionedfiles object should provide a different
1502
1466
# method to get the number of keys.
1512
1476
:param using: If True, list only branches using this repository.
1514
1478
if using and not self.is_shared():
1515
return self.bzrdir.list_branches()
1480
return [self.bzrdir.open_branch()]
1481
except errors.NotBranchError:
1516
1483
class Evaluator(object):
1518
1485
def __init__(self):
1527
1494
except errors.NoRepositoryPresent:
1530
return False, ([], repository)
1497
return False, (None, repository)
1531
1498
self.first_call = False
1532
value = (bzrdir.list_branches(), None)
1500
value = (bzrdir.open_branch(), None)
1501
except errors.NotBranchError:
1502
value = (None, None)
1533
1503
return True, value
1536
for branches, repository in bzrdir.BzrDir.find_bzrdirs(
1537
self.user_transport, evaluate=Evaluator()):
1538
if branches is not None:
1539
ret.extend(branches)
1506
for branch, repository in bzrdir.BzrDir.find_bzrdirs(
1507
self.bzrdir.root_transport, evaluate=Evaluator()):
1508
if branch is not None:
1509
branches.append(branch)
1540
1510
if not using and repository is not None:
1541
ret.extend(repository.find_branches())
1511
branches.extend(repository.find_branches())
1544
1514
@needs_read_lock
1545
1515
def search_missing_revision_ids(self, other, revision_id=None, find_ghosts=True):
1923
1892
rev = self._serializer.read_revision_from_string(text)
1924
1893
yield (revid, rev)
1896
def get_revision_xml(self, revision_id):
1897
# TODO: jam 20070210 This shouldn't be necessary since get_revision
1898
# would have already do it.
1899
# TODO: jam 20070210 Just use _serializer.write_revision_to_string()
1900
# TODO: this can't just be replaced by:
1901
# return self._serializer.write_revision_to_string(
1902
# self.get_revision(revision_id))
1903
# as cStringIO preservers the encoding unlike write_revision_to_string
1904
# or some other call down the path.
1905
rev = self.get_revision(revision_id)
1906
rev_tmp = cStringIO.StringIO()
1907
# the current serializer..
1908
self._serializer.write_revision(rev, rev_tmp)
1910
return rev_tmp.getvalue()
1926
1912
def get_deltas_for_revisions(self, revisions, specific_fileids=None):
1927
1913
"""Produce a generator of revision deltas.
2171
2157
selected_keys = set((revid,) for revid in revision_ids)
2172
2158
w = _inv_weave or self.inventories
2173
return self._find_file_ids_from_xml_inventory_lines(
2174
w.iter_lines_added_or_present_in_keys(
2175
selected_keys, pb=None),
2159
pb = ui.ui_factory.nested_progress_bar()
2161
return self._find_file_ids_from_xml_inventory_lines(
2162
w.iter_lines_added_or_present_in_keys(
2163
selected_keys, pb=pb),
2178
2168
def iter_files_bytes(self, desired_files):
2179
2169
"""Iterate through file versions.
2340
2330
num_file_ids = len(file_ids)
2341
2331
for file_id, altered_versions in file_ids.iteritems():
2342
2332
if pb is not None:
2343
pb.update("Fetch texts", count, num_file_ids)
2333
pb.update("fetch texts", count, num_file_ids)
2345
2335
yield ("file", file_id, altered_versions)
2389
2379
"""single-document based inventory iteration."""
2390
2380
inv_xmls = self._iter_inventory_xmls(revision_ids, ordering)
2391
2381
for text, revision_id in inv_xmls:
2392
yield self._deserialise_inventory(revision_id, text)
2382
yield self.deserialise_inventory(revision_id, text)
2394
2384
def _iter_inventory_xmls(self, revision_ids, ordering):
2395
2385
if ordering is None:
2427
2417
next_key = None
2430
def _deserialise_inventory(self, revision_id, xml):
2420
def deserialise_inventory(self, revision_id, xml):
2431
2421
"""Transform the xml into an inventory object.
2433
2423
:param revision_id: The expected revision id of the inventory.
2434
2424
:param xml: A serialised inventory.
2436
2426
result = self._serializer.read_inventory_from_string(xml, revision_id,
2437
entry_cache=self._inventory_entry_cache,
2438
return_from_cache=self._safe_to_return_from_cache)
2427
entry_cache=self._inventory_entry_cache)
2439
2428
if result.revision_id != revision_id:
2440
2429
raise AssertionError('revision id mismatch %s != %s' % (
2441
2430
result.revision_id, revision_id))
2433
def serialise_inventory(self, inv):
2434
return self._serializer.write_inventory_to_string(inv)
2436
def _serialise_inventory_to_lines(self, inv):
2437
return self._serializer.write_inventory_to_lines(inv)
2444
2439
def get_serializer_format(self):
2445
2440
return self._serializer.format_num
2447
2442
@needs_read_lock
2448
def _get_inventory_xml(self, revision_id):
2449
"""Get serialized inventory as a string."""
2443
def get_inventory_xml(self, revision_id):
2444
"""Get inventory XML as a file object."""
2450
2445
texts = self._iter_inventory_xmls([revision_id], 'unordered')
2452
2447
text, revision_id = texts.next()
2454
2449
raise errors.HistoryMissing(self, 'inventory', revision_id)
2453
def get_inventory_sha1(self, revision_id):
2454
"""Return the sha1 hash of the inventory entry
2456
return self.get_revision(revision_id).inventory_sha1
2457
2458
def get_rev_id_for_revno(self, revno, known_pair):
2458
2459
"""Return the revision id of a revno, given a later (revno, revid)
2459
2460
pair in the same history.
2511
2512
next_id = parents[0]
2515
def get_revision_inventory(self, revision_id):
2516
"""Return inventory of a past revision."""
2517
# TODO: Unify this with get_inventory()
2518
# bzr 0.0.6 and later imposes the constraint that the inventory_id
2519
# must be the same as its revision, so this is trivial.
2520
if revision_id is None:
2521
# This does not make sense: if there is no revision,
2522
# then it is the current tree inventory surely ?!
2523
# and thus get_root_id() is something that looks at the last
2524
# commit on the branch, and the get_root_id is an inventory check.
2525
raise NotImplementedError
2526
# return Inventory(self.get_root_id())
2528
return self.get_inventory(revision_id)
2513
2530
def is_shared(self):
2514
2531
"""Return True if this repository is flagged as a shared repository."""
2515
2532
raise NotImplementedError(self.is_shared)
2549
2566
return RevisionTree(self, Inventory(root_id=None),
2550
2567
_mod_revision.NULL_REVISION)
2552
inv = self.get_inventory(revision_id)
2569
inv = self.get_revision_inventory(revision_id)
2553
2570
return RevisionTree(self, inv, revision_id)
2555
2572
def revision_trees(self, revision_ids):
2608
2625
keys = tsort.topo_sort(parent_map)
2609
2626
return [None] + list(keys)
2611
def pack(self, hint=None, clean_obsolete_packs=False):
2628
def pack(self, hint=None):
2612
2629
"""Compress the data within the repository.
2614
2631
This operation only makes sense for some repository types. For other
2624
2641
obtained from the result of commit_write_group(). Out of
2625
2642
date hints are simply ignored, because concurrent operations
2626
2643
can obsolete them rapidly.
2628
:param clean_obsolete_packs: Clean obsolete packs immediately after
2632
2646
def get_transaction(self):
2648
2662
for ((revision_id,), parent_keys) in \
2649
2663
self.revisions.get_parent_map(query_keys).iteritems():
2650
2664
if parent_keys:
2651
result[revision_id] = tuple([parent_revid
2652
for (parent_revid,) in parent_keys])
2665
result[revision_id] = tuple(parent_revid
2666
for (parent_revid,) in parent_keys)
2654
2668
result[revision_id] = (_mod_revision.NULL_REVISION,)
2657
2671
def _make_parents_provider(self):
2661
def get_known_graph_ancestry(self, revision_ids):
2662
"""Return the known graph for a set of revision ids and their ancestors.
2664
st = static_tuple.StaticTuple
2665
revision_keys = [st(r_id).intern() for r_id in revision_ids]
2666
known_graph = self.revisions.get_known_graph_ancestry(revision_keys)
2667
return graph.GraphThunkIdsToKeys(known_graph)
2669
2674
def get_graph(self, other_repository=None):
2670
2675
"""Return the graph walker for this repository format"""
2671
2676
parents_provider = self._make_parents_provider()
2766
2771
result.check(callback_refs)
2769
def _warn_if_deprecated(self, branch=None):
2774
def _warn_if_deprecated(self):
2770
2775
global _deprecation_warning_done
2771
2776
if _deprecation_warning_done:
2775
conf = config.GlobalConfig()
2777
conf = branch.get_config()
2778
if conf.suppress_warning('format_deprecation'):
2780
warning("Format %s for %s is deprecated -"
2781
" please use 'bzr upgrade' to get better performance"
2782
% (self._format, self.bzrdir.transport.base))
2784
_deprecation_warning_done = True
2778
_deprecation_warning_done = True
2779
warning("Format %s for %s is deprecated - please use 'bzr upgrade' to get better performance"
2780
% (self._format, self.bzrdir.transport.base))
2786
2782
def supports_rich_root(self):
2787
2783
return self._format.rich_root_data
3070
3066
pack_compresses = False
3071
3067
# Does the repository inventory storage understand references to trees?
3072
3068
supports_tree_reference = None
3073
# Is the format experimental ?
3074
experimental = False
3077
return "%s()" % self.__class__.__name__
3071
return "<%s>" % self.__class__.__name__
3079
3073
def __eq__(self, other):
3080
3074
# format objects are generally stateless
3095
3089
transport = a_bzrdir.get_repository_transport(None)
3096
format_string = transport.get_bytes("format")
3090
format_string = transport.get("format").read()
3097
3091
return format_registry.get(format_string)
3098
3092
except errors.NoSuchFile:
3099
3093
raise errors.NoRepositoryPresent(a_bzrdir)
3199
3193
raise NotImplementedError(self.open)
3201
def _run_post_repo_init_hooks(self, repository, a_bzrdir, shared):
3202
from bzrlib.bzrdir import BzrDir, RepoInitHookParams
3203
hooks = BzrDir.hooks['post_repo_init']
3206
params = RepoInitHookParams(repository, self, a_bzrdir, shared)
3211
3196
class MetaDirRepositoryFormat(RepositoryFormat):
3212
3197
"""Common base class for the new repositories using the metadir layout."""
3418
3403
:param revision_id: if None all content is copied, if NULL_REVISION no
3419
3404
content is copied.
3405
:param pb: optional progress bar to use for progress reports. If not
3406
provided a default one will be created.
3423
ui.ui_factory.warn_experimental_format_fetch(self)
3424
3409
from bzrlib.fetch import RepoFetcher
3425
# See <https://launchpad.net/bugs/456077> asking for a warning here
3426
if self.source._format.network_name() != self.target._format.network_name():
3427
ui.ui_factory.show_user_warning('cross_format_fetch',
3428
from_format=self.source._format,
3429
to_format=self.target._format)
3430
3410
f = RepoFetcher(to_repository=self.target,
3431
3411
from_repository=self.source,
3432
3412
last_revision=revision_id,
3433
3413
fetch_spec=fetch_spec,
3434
find_ghosts=find_ghosts)
3414
pb=pb, find_ghosts=find_ghosts)
3436
3416
def _walk_to_common_revisions(self, revision_ids):
3437
3417
"""Walk out from revision_ids in source to revisions target has.
3606
3586
self.target.texts.insert_record_stream(
3607
3587
self.source.texts.get_record_stream(
3608
3588
self.source.texts.keys(), 'topological', False))
3609
pb.update('Copying inventory', 0, 1)
3589
pb.update('copying inventory', 0, 1)
3610
3590
self.target.inventories.insert_record_stream(
3611
3591
self.source.inventories.get_record_stream(
3612
3592
self.source.inventories.keys(), 'topological', False))
3833
3813
basis_id, delta, current_revision_id, parents_parents)
3834
3814
cache[current_revision_id] = parent_tree
3836
def _fetch_batch(self, revision_ids, basis_id, cache, a_graph=None):
3816
def _fetch_batch(self, revision_ids, basis_id, cache):
3837
3817
"""Fetch across a few revisions.
3839
3819
:param revision_ids: The revisions to copy
3840
3820
:param basis_id: The revision_id of a tree that must be in cache, used
3841
3821
as a basis for delta when no other base is available
3842
3822
:param cache: A cache of RevisionTrees that we can use.
3843
:param a_graph: A Graph object to determine the heads() of the
3844
rich-root data stream.
3845
3823
:return: The revision_id of the last converted tree. The RevisionTree
3846
3824
for it will be in cache
3854
3832
pending_revisions = []
3855
3833
parent_map = self.source.get_parent_map(revision_ids)
3856
3834
self._fetch_parent_invs_for_stacking(parent_map, cache)
3857
self.source._safe_to_return_from_cache = True
3858
3835
for tree in self.source.revision_trees(revision_ids):
3859
3836
# Find a inventory delta for this revision.
3860
3837
# Find text entries that need to be copied, too.
3908
3885
pending_revisions.append(revision)
3909
3886
cache[current_revision_id] = tree
3910
3887
basis_id = current_revision_id
3911
self.source._safe_to_return_from_cache = False
3912
3888
# Copy file texts
3913
3889
from_texts = self.source.texts
3914
3890
to_texts = self.target.texts
3915
3891
if root_keys_to_create:
3916
root_stream = _mod_fetch._new_root_data_stream(
3892
from bzrlib.fetch import _new_root_data_stream
3893
root_stream = _new_root_data_stream(
3917
3894
root_keys_to_create, self._revision_id_to_root_id, parent_map,
3918
self.source, graph=a_graph)
3919
3896
to_texts.insert_record_stream(root_stream)
3920
3897
to_texts.insert_record_stream(from_texts.get_record_stream(
3921
3898
text_keys, self.target._format._fetch_order,
3978
3955
cache[basis_id] = basis_tree
3979
3956
del basis_tree # We don't want to hang on to it here
3981
if self._converting_to_rich_root and len(revision_ids) > 100:
3982
a_graph = _mod_fetch._get_rich_root_heads_graph(self.source,
3987
3958
for offset in range(0, len(revision_ids), batch_size):
3988
3959
self.target.start_write_group()
3990
3961
pb.update('Transferring revisions', offset,
3991
3962
len(revision_ids))
3992
3963
batch = revision_ids[offset:offset+batch_size]
3993
basis_id = self._fetch_batch(batch, basis_id, cache,
3964
basis_id = self._fetch_batch(batch, basis_id, cache)
3996
self.source._safe_to_return_from_cache = False
3997
3966
self.target.abort_write_group()
4011
3980
"""See InterRepository.fetch()."""
4012
3981
if fetch_spec is not None:
4013
3982
raise AssertionError("Not implemented yet...")
4014
ui.ui_factory.warn_experimental_format_fetch(self)
3983
# See <https://launchpad.net/bugs/456077> asking for a warning here
3985
# nb this is only active for local-local fetches; other things using
3987
ui.ui_factory.warn_cross_format_fetch(self.source._format,
3988
self.target._format)
4015
3989
if (not self.source.supports_rich_root()
4016
3990
and self.target.supports_rich_root()):
4017
3991
self._converting_to_rich_root = True
4018
3992
self._revision_id_to_root_id = {}
4020
3994
self._converting_to_rich_root = False
4021
# See <https://launchpad.net/bugs/456077> asking for a warning here
4022
if self.source._format.network_name() != self.target._format.network_name():
4023
ui.ui_factory.show_user_warning('cross_format_fetch',
4024
from_format=self.source._format,
4025
to_format=self.target._format)
4026
3995
revision_ids = self.target.search_missing_revision_ids(self.source,
4027
3996
revision_id, find_ghosts=find_ghosts).get_keys()
4028
3997
if not revision_ids:
4097
4066
:param to_convert: The disk object to convert.
4098
4067
:param pb: a progress bar to use for progress information.
4100
pb = ui.ui_factory.nested_progress_bar()
4103
4072
# this is only useful with metadir layouts - separated repo content.
4104
4073
# trigger an assertion if not such
4105
4074
repo._format.get_format_string()
4106
4075
self.repo_dir = repo.bzrdir
4107
pb.update('Moving repository to repository.backup')
4076
self.step('Moving repository to repository.backup')
4108
4077
self.repo_dir.transport.move('repository', 'repository.backup')
4109
4078
backup_transport = self.repo_dir.transport.clone('repository.backup')
4110
4079
repo._format.check_conversion_target(self.target_format)
4111
4080
self.source_repo = repo._format.open(self.repo_dir,
4113
4082
_override_transport=backup_transport)
4114
pb.update('Creating new repository')
4083
self.step('Creating new repository')
4115
4084
converted = self.target_format.initialize(self.repo_dir,
4116
4085
self.source_repo.is_shared())
4117
4086
converted.lock_write()
4119
pb.update('Copying content')
4088
self.step('Copying content into repository.')
4120
4089
self.source_repo.copy_content_into(converted)
4122
4091
converted.unlock()
4123
pb.update('Deleting old repository content')
4092
self.step('Deleting old repository content.')
4124
4093
self.repo_dir.transport.delete_tree('repository.backup')
4125
ui.ui_factory.note('repository converted')
4094
self.pb.note('repository converted')
4096
def step(self, message):
4097
"""Update the pb by a step."""
4099
self.pb.update(message, self.count, self.total)
4129
4102
_unescape_map = {
4311
4284
self._extract_and_insert_inventories(
4312
4285
substream, src_serializer)
4313
4286
elif substream_type == 'inventory-deltas':
4287
ui.ui_factory.warn_cross_format_fetch(src_format,
4288
self.target_repo._format)
4314
4289
self._extract_and_insert_inventory_deltas(
4315
4290
substream, src_serializer)
4316
4291
elif substream_type == 'chk_bytes':
4350
4325
if versioned_file is None:
4352
# TODO: key is often going to be a StaticTuple object
4353
# I don't believe we can define a method by which
4354
# (prefix,) + StaticTuple will work, though we could
4355
# define a StaticTuple.sq_concat that would allow you to
4356
# pass in either a tuple or a StaticTuple as the second
4357
# object, so instead we could have:
4358
# StaticTuple(prefix) + key here...
4359
4327
missing_keys.update((prefix,) + key for key in
4360
4328
versioned_file.get_missing_compression_parent_keys())
4361
4329
except NotImplementedError:
4473
4441
fetching the inventory weave.
4475
4443
if self._rich_root_upgrade():
4476
return _mod_fetch.Inter1and2Helper(
4445
return bzrlib.fetch.Inter1and2Helper(
4477
4446
self.from_repository).generate_root_texts(revs)
added
removed
bzrlib/repository.py
