264
265
# return '%X.%X' % (int(st.st_mtime), st.st_mode)
268
def _unpack_stat(packed_stat):
269
"""Turn a packed_stat back into the stat fields.
271
This is meant as a debugging tool, should not be used in real code.
273
(st_size, st_mtime, st_ctime, st_dev, st_ino,
274
st_mode) = struct.unpack('>LLLLLL', binascii.a2b_base64(packed_stat))
275
return dict(st_size=st_size, st_mtime=st_mtime, st_ctime=st_ctime,
276
st_dev=st_dev, st_ino=st_ino, st_mode=st_mode)
267
279
class SHA1Provider(object):
268
280
"""An interface for getting sha1s of a file."""
363
376
HEADER_FORMAT_2 = '#bazaar dirstate flat format 2\n'
364
377
HEADER_FORMAT_3 = '#bazaar dirstate flat format 3\n'
366
def __init__(self, path, sha1_provider):
379
def __init__(self, path, sha1_provider, worth_saving_limit=0):
367
380
"""Create a DirState object.
369
382
:param path: The path at which the dirstate file on disk should live.
370
383
:param sha1_provider: an object meeting the SHA1Provider interface.
384
:param worth_saving_limit: when the exact number of hash changed
385
entries is known, only bother saving the dirstate if more than
386
this count of entries have changed.
387
-1 means never save hash changes, 0 means always save hash changes.
372
389
# _header_state and _dirblock_state represent the current state
373
390
# of the dirstate metadata and the per-row data respectiely.
411
428
self._last_block_index = None
412
429
self._last_entry_index = None
430
# The set of known hash changes
431
self._known_hash_changes = set()
432
# How many hash changed entries can we have without saving
433
self._worth_saving_limit = worth_saving_limit
414
435
def __repr__(self):
415
436
return "%s(%r)" % \
416
437
(self.__class__.__name__, self._filename)
439
def _mark_modified(self, hash_changed_entries=None, header_modified=False):
440
"""Mark this dirstate as modified.
442
:param hash_changed_entries: if non-None, mark just these entries as
443
having their hash modified.
444
:param header_modified: mark the header modified as well, not just the
447
#trace.mutter_callsite(3, "modified hash entries: %s", hash_changed_entries)
448
if hash_changed_entries:
449
self._known_hash_changes.update([e[0] for e in hash_changed_entries])
450
if self._dirblock_state in (DirState.NOT_IN_MEMORY,
451
DirState.IN_MEMORY_UNMODIFIED):
452
# If the dirstate is already marked a IN_MEMORY_MODIFIED, then
453
# that takes precedence.
454
self._dirblock_state = DirState.IN_MEMORY_HASH_MODIFIED
456
# TODO: Since we now have a IN_MEMORY_HASH_MODIFIED state, we
457
# should fail noisily if someone tries to set
458
# IN_MEMORY_MODIFIED but we don't have a write-lock!
459
# We don't know exactly what changed so disable smart saving
460
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
462
self._header_state = DirState.IN_MEMORY_MODIFIED
464
def _mark_unmodified(self):
465
"""Mark this dirstate as unmodified."""
466
self._header_state = DirState.IN_MEMORY_UNMODIFIED
467
self._dirblock_state = DirState.IN_MEMORY_UNMODIFIED
468
self._known_hash_changes = set()
418
470
def add(self, path, file_id, kind, stat, fingerprint):
419
471
"""Add a path to be tracked.
546
598
if kind == 'directory':
547
599
# insert a new dirblock
548
600
self._ensure_block(block_index, entry_index, utf8path)
549
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
601
self._mark_modified()
550
602
if self._id_index:
551
self._id_index.setdefault(entry_key[2], set()).add(entry_key)
603
self._add_to_id_index(self._id_index, entry_key)
553
605
def _bisect(self, paths):
554
606
"""Bisect through the disk structure for specific rows.
1019
1071
self._ghosts = []
1020
1072
self._parents = [parents[0]]
1021
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
1022
self._header_state = DirState.IN_MEMORY_MODIFIED
1073
self._mark_modified(header_modified=True)
1024
1075
def _empty_parent_info(self):
1025
1076
return [DirState.NULL_PARENT_DETAILS] * (len(self._parents) -
1567
1617
id_index = self._get_id_index()
1568
1618
for file_id in new_ids:
1569
for key in id_index.get(file_id, []):
1619
for key in id_index.get(file_id, ()):
1570
1620
block_i, entry_i, d_present, f_present = \
1571
1621
self._get_block_entry_index(key[0], key[1], tree_index)
1572
1622
if not f_present:
1733
1783
self._sha_cutoff_time()
1734
1784
if (stat_value.st_mtime < self._cutoff_time
1735
1785
and stat_value.st_ctime < self._cutoff_time):
1736
entry[1][0] = ('f', sha1, entry[1][0][2], entry[1][0][3],
1738
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
1786
entry[1][0] = ('f', sha1, stat_value.st_size, entry[1][0][3],
1788
self._mark_modified([entry])
1740
1790
def _sha_cutoff_time(self):
1741
1791
"""Return cutoff time.
1799
1849
"""Serialise the entire dirstate to a sequence of lines."""
1800
1850
if (self._header_state == DirState.IN_MEMORY_UNMODIFIED and
1801
1851
self._dirblock_state == DirState.IN_MEMORY_UNMODIFIED):
1802
# read whats on disk.
1852
# read what's on disk.
1803
1853
self._state_file.seek(0)
1804
1854
return self._state_file.readlines()
1806
1856
lines.append(self._get_parents_line(self.get_parent_ids()))
1807
1857
lines.append(self._get_ghosts_line(self._ghosts))
1808
# append the root line which is special cased
1809
lines.extend(map(self._entry_to_line, self._iter_entries()))
1858
lines.extend(self._get_entry_lines())
1810
1859
return self._get_output_lines(lines)
1812
1861
def _get_ghosts_line(self, ghost_ids):
1817
1866
"""Create a line for the state file for parents information."""
1818
1867
return '\0'.join([str(len(parent_ids))] + parent_ids)
1869
def _get_entry_lines(self):
1870
"""Create lines for entries."""
1871
return map(self._entry_to_line, self._iter_entries())
1820
1873
def _get_fields_to_entry(self):
1821
1874
"""Get a function which converts entry fields into a entry record.
1980
2033
' tree_index, file_id and path')
1983
possible_keys = self._get_id_index().get(fileid_utf8, None)
2036
possible_keys = self._get_id_index().get(fileid_utf8, ())
1984
2037
if not possible_keys:
1985
2038
return None, None
1986
2039
for key in possible_keys:
2145
2198
def _get_id_index(self):
2146
"""Get an id index of self._dirblocks."""
2199
"""Get an id index of self._dirblocks.
2201
This maps from file_id => [(directory, name, file_id)] entries where
2202
that file_id appears in one of the trees.
2147
2204
if self._id_index is None:
2149
2206
for key, tree_details in self._iter_entries():
2150
id_index.setdefault(key[2], set()).add(key)
2207
self._add_to_id_index(id_index, key)
2151
2208
self._id_index = id_index
2152
2209
return self._id_index
2211
def _add_to_id_index(self, id_index, entry_key):
2212
"""Add this entry to the _id_index mapping."""
2213
# This code used to use a set for every entry in the id_index. However,
2214
# it is *rare* to have more than one entry. So a set is a large
2215
# overkill. And even when we do, we won't ever have more than the
2216
# number of parent trees. Which is still a small number (rarely >2). As
2217
# such, we use a simple tuple, and do our own uniqueness checks. While
2218
# the 'in' check is O(N) since N is nicely bounded it shouldn't ever
2219
# cause quadratic failure.
2220
# TODO: This should use StaticTuple
2221
file_id = entry_key[2]
2222
entry_key = static_tuple.StaticTuple.from_sequence(entry_key)
2223
if file_id not in id_index:
2224
id_index[file_id] = static_tuple.StaticTuple(entry_key,)
2226
entry_keys = id_index[file_id]
2227
if entry_key not in entry_keys:
2228
id_index[file_id] = entry_keys + (entry_key,)
2230
def _remove_from_id_index(self, id_index, entry_key):
2231
"""Remove this entry from the _id_index mapping.
2233
It is an programming error to call this when the entry_key is not
2236
file_id = entry_key[2]
2237
entry_keys = list(id_index[file_id])
2238
entry_keys.remove(entry_key)
2239
id_index[file_id] = static_tuple.StaticTuple.from_sequence(entry_keys)
2154
2241
def _get_output_lines(self, lines):
2155
2242
"""Format lines for final output.
2176
2263
"""The number of parent entries in each record row."""
2177
2264
return len(self._parents) - len(self._ghosts)
2180
def on_file(path, sha1_provider=None):
2267
def on_file(cls, path, sha1_provider=None, worth_saving_limit=0):
2181
2268
"""Construct a DirState on the file at path "path".
2183
2270
:param path: The path at which the dirstate file on disk should live.
2184
2271
:param sha1_provider: an object meeting the SHA1Provider interface.
2185
2272
If None, a DefaultSHA1Provider is used.
2273
:param worth_saving_limit: when the exact number of hash changed
2274
entries is known, only bother saving the dirstate if more than
2275
this count of entries have changed. -1 means never save.
2186
2276
:return: An unlocked DirState object, associated with the given path.
2188
2278
if sha1_provider is None:
2189
2279
sha1_provider = DefaultSHA1Provider()
2190
result = DirState(path, sha1_provider)
2280
result = cls(path, sha1_provider,
2281
worth_saving_limit=worth_saving_limit)
2193
2284
def _read_dirblocks_if_needed(self):
2285
2376
trace.mutter('Not saving DirState because '
2286
2377
'_changes_aborted is set.')
2288
if (self._header_state == DirState.IN_MEMORY_MODIFIED or
2289
self._dirblock_state == DirState.IN_MEMORY_MODIFIED):
2379
# TODO: Since we now distinguish IN_MEMORY_MODIFIED from
2380
# IN_MEMORY_HASH_MODIFIED, we should only fail quietly if we fail
2381
# to save an IN_MEMORY_HASH_MODIFIED, and fail *noisily* if we
2382
# fail to save IN_MEMORY_MODIFIED
2383
if self._worth_saving():
2291
2384
grabbed_write_lock = False
2292
2385
if self._lock_state != 'w':
2293
2386
grabbed_write_lock, new_lock = self._lock_token.temporary_write_lock()
2301
2394
# We couldn't grab a write lock, so we switch back to a read one
2397
lines = self.get_lines()
2304
2398
self._state_file.seek(0)
2305
self._state_file.writelines(self.get_lines())
2399
self._state_file.writelines(lines)
2306
2400
self._state_file.truncate()
2307
2401
self._state_file.flush()
2308
self._header_state = DirState.IN_MEMORY_UNMODIFIED
2309
self._dirblock_state = DirState.IN_MEMORY_UNMODIFIED
2402
self._mark_unmodified()
2311
2404
if grabbed_write_lock:
2312
2405
self._lock_token = self._lock_token.restore_read_lock()
2315
2408
# not changed contents. Since restore_read_lock may
2316
2409
# not be an atomic operation.
2411
def _worth_saving(self):
2412
"""Is it worth saving the dirstate or not?"""
2413
if (self._header_state == DirState.IN_MEMORY_MODIFIED
2414
or self._dirblock_state == DirState.IN_MEMORY_MODIFIED):
2416
if self._dirblock_state == DirState.IN_MEMORY_HASH_MODIFIED:
2417
if self._worth_saving_limit == -1:
2418
# We never save hash changes when the limit is -1
2420
# If we're using smart saving and only a small number of
2421
# entries have changed their hash, don't bother saving. John has
2422
# suggested using a heuristic here based on the size of the
2423
# changed files and/or tree. For now, we go with a configurable
2424
# number of changes, keeping the calculation time
2425
# as low overhead as possible. (This also keeps all existing
2426
# tests passing as the default is 0, i.e. always save.)
2427
if len(self._known_hash_changes) >= self._worth_saving_limit:
2318
2431
def _set_data(self, parent_ids, dirblocks):
2319
2432
"""Set the full dirstate data in memory.
2329
2442
# our memory copy is now authoritative.
2330
2443
self._dirblocks = dirblocks
2331
self._header_state = DirState.IN_MEMORY_MODIFIED
2332
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
2444
self._mark_modified(header_modified=True)
2333
2445
self._parents = list(parent_ids)
2334
2446
self._id_index = None
2335
2447
self._packed_stat_index = None
2355
2467
self._make_absent(entry)
2356
2468
self.update_minimal(('', '', new_id), 'd',
2357
2469
path_utf8='', packed_stat=entry[1][0][4])
2358
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
2470
self._mark_modified()
2471
# XXX: This was added by Ian, we need to make sure there
2472
# are tests for it, because it isn't in bzr.dev TRUNK
2473
# It looks like the only place it is called is in setting the root
2474
# id of the tree. So probably we never had an _id_index when we
2475
# don't even have a root yet.
2476
if self._id_index is not None:
2477
self._add_to_id_index(self._id_index, entry[0])
2360
2479
def set_parent_trees(self, trees, ghosts):
2361
2480
"""Set the parent trees for the dirstate.
2415
2534
by_path[entry[0]] = [entry[1][0]] + \
2416
2535
[DirState.NULL_PARENT_DETAILS] * parent_count
2417
id_index[entry[0][2]] = set([entry[0]])
2536
# TODO: Possibly inline this, since we know it isn't present yet
2537
# id_index[entry[0][2]] = (entry[0],)
2538
self._add_to_id_index(id_index, entry[0])
2419
2540
# now the parent trees:
2420
2541
for tree_index, tree in enumerate(parent_trees):
2426
2547
# the suffix is from tree_index+1:parent_count+1.
2427
2548
new_location_suffix = [DirState.NULL_PARENT_DETAILS] * (parent_count - tree_index)
2428
2549
# now stitch in all the entries from this tree
2429
for path, entry in tree.inventory.iter_entries_by_dir():
2550
for path, entry in tree.iter_entries_by_dir():
2430
2551
# here we process each trees details for each item in the tree.
2431
2552
# we first update any existing entries for the id at other paths,
2432
2553
# then we either create or update the entry for the id at the
2442
2563
new_entry_key = (dirname, basename, file_id)
2443
2564
# tree index consistency: All other paths for this id in this tree
2444
2565
# index must point to the correct path.
2445
for entry_key in id_index.setdefault(file_id, set()):
2566
for entry_key in id_index.get(file_id, ()):
2446
2567
# TODO:PROFILING: It might be faster to just update
2447
2568
# rather than checking if we need to, and then overwrite
2448
2569
# the one we are located at.
2454
2575
by_path[entry_key][tree_index] = ('r', path_utf8, 0, False, '')
2455
2576
# by path consistency: Insert into an existing path record (trivial), or
2456
2577
# add a new one with relocation pointers for the other tree indexes.
2457
if new_entry_key in id_index[file_id]:
2578
entry_keys = id_index.get(file_id, ())
2579
if new_entry_key in entry_keys:
2458
2580
# there is already an entry where this data belongs, just insert it.
2459
2581
by_path[new_entry_key][tree_index] = \
2460
2582
self._inv_entry_to_details(entry)
2465
2587
new_details = []
2466
2588
for lookup_index in xrange(tree_index):
2467
2589
# boundary case: this is the first occurence of file_id
2468
# so there are no id_indexs, possibly take this out of
2590
# so there are no id_indexes, possibly take this out of
2470
if not len(id_index[file_id]):
2592
if not len(entry_keys):
2471
2593
new_details.append(DirState.NULL_PARENT_DETAILS)
2473
2595
# grab any one entry, use it to find the right path.
2474
2596
# TODO: optimise this to reduce memory use in highly
2475
2597
# fragmented situations by reusing the relocation
2477
a_key = iter(id_index[file_id]).next()
2599
a_key = iter(entry_keys).next()
2478
2600
if by_path[a_key][lookup_index][0] in ('r', 'a'):
2479
2601
# its a pointer or missing statement, use it as is.
2480
2602
new_details.append(by_path[a_key][lookup_index])
2485
2607
new_details.append(self._inv_entry_to_details(entry))
2486
2608
new_details.extend(new_location_suffix)
2487
2609
by_path[new_entry_key] = new_details
2488
id_index[file_id].add(new_entry_key)
2610
self._add_to_id_index(id_index, new_entry_key)
2489
2611
# --- end generation of full tree mappings
2491
2613
# sort and output all the entries
2493
2615
self._entries_to_current_state(new_entries)
2494
2616
self._parents = [rev_id for rev_id, tree in trees]
2495
2617
self._ghosts = list(ghosts)
2496
self._header_state = DirState.IN_MEMORY_MODIFIED
2497
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
2618
self._mark_modified(header_modified=True)
2498
2619
self._id_index = id_index
2500
2621
def _sort_entries(self, entry_list):
2637
2758
current_old[0][1].decode('utf8'))
2638
2759
self._make_absent(current_old)
2639
2760
current_old = advance(old_iterator)
2640
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
2761
self._mark_modified()
2641
2762
self._id_index = None
2642
2763
self._packed_stat_index = None
2644
2765
trace.mutter("set_state_from_inventory complete.")
2767
def set_state_from_scratch(self, working_inv, parent_trees, parent_ghosts):
2768
"""Wipe the currently stored state and set it to something new.
2770
This is a hard-reset for the data we are working with.
2772
# Technically, we really want a write lock, but until we write, we
2773
# don't really need it.
2774
self._requires_lock()
2775
# root dir and root dir contents with no children. We have to have a
2776
# root for set_state_from_inventory to work correctly.
2777
empty_root = (('', '', inventory.ROOT_ID),
2778
[('d', '', 0, False, DirState.NULLSTAT)])
2779
empty_tree_dirblocks = [('', [empty_root]), ('', [])]
2780
self._set_data([], empty_tree_dirblocks)
2781
self.set_state_from_inventory(working_inv)
2782
self.set_parent_trees(parent_trees, parent_ghosts)
2646
2784
def _make_absent(self, current_old):
2647
2785
"""Mark current_old - an entry - as absent for tree 0.
2673
2811
block[1].pop(entry_index)
2674
2812
# if we have an id_index in use, remove this key from it for this id.
2675
2813
if self._id_index is not None:
2676
self._id_index[current_old[0][2]].remove(current_old[0])
2814
self._remove_from_id_index(self._id_index, current_old[0])
2677
2815
# update all remaining keys for this id to record it as absent. The
2678
2816
# existing details may either be the record we are marking as deleted
2679
2817
# (if there were other trees with the id present at this path), or may
2692
2830
if update_tree_details[0][0] == 'a': # absent
2693
2831
raise AssertionError('bad row %r' % (update_tree_details,))
2694
2832
update_tree_details[0] = DirState.NULL_PARENT_DETAILS
2695
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
2833
self._mark_modified()
2696
2834
return last_reference
2698
2836
def update_minimal(self, key, minikind, executable=False, fingerprint='',
2750
2888
# new entry, synthesis cross reference here,
2751
existing_keys = id_index.setdefault(key[2], set())
2889
existing_keys = id_index.get(key[2], ())
2752
2890
if not existing_keys:
2753
2891
# not currently in the state, simplest case
2754
2892
new_entry = key, [new_details] + self._empty_parent_info()
2786
2924
other_entry = other_block[other_entry_index]
2787
2925
other_entry[1][0] = ('r', path_utf8, 0, False, '')
2788
self._maybe_remove_row(other_block, other_entry_index,
2926
if self._maybe_remove_row(other_block, other_entry_index,
2928
# If the row holding this was removed, we need to
2929
# recompute where this entry goes
2930
entry_index, _ = self._find_entry_index(key, block)
2792
2933
# adds a tuple to the new details for each column
2794
2935
# - or by creating a new pointer to the right row inside that column
2795
2936
num_present_parents = self._num_present_parents()
2796
2937
if num_present_parents:
2938
# TODO: This re-evaluates the existing_keys set, do we need
2939
# to do that ourselves?
2797
2940
other_key = list(existing_keys)[0]
2798
2941
for lookup_index in xrange(1, num_present_parents + 1):
2799
2942
# grab any one entry, use it to find the right path.
2818
2961
pointer_path = osutils.pathjoin(*other_key[0:2])
2819
2962
new_entry[1].append(('r', pointer_path, 0, False, ''))
2820
2963
block.insert(entry_index, new_entry)
2821
existing_keys.add(key)
2964
self._add_to_id_index(id_index, key)
2823
2966
# Does the new state matter?
2824
2967
block[entry_index][1][0] = new_details
2833
2976
# converted to relocated.
2834
2977
if path_utf8 is None:
2835
2978
raise AssertionError('no path')
2836
for entry_key in id_index.setdefault(key[2], set()):
2979
existing_keys = id_index.get(key[2], ())
2980
if key not in existing_keys:
2981
raise AssertionError('We found the entry in the blocks, but'
2982
' the key is not in the id_index.'
2983
' key: %s, existing_keys: %s' % (key, existing_keys))
2984
for entry_key in existing_keys:
2837
2985
# TODO:PROFILING: It might be faster to just update
2838
2986
# rather than checking if we need to, and then overwrite
2839
2987
# the one we are located at.
2857
3005
if not present:
2858
3006
self._dirblocks.insert(block_index, (subdir_key[0], []))
2860
self._dirblock_state = DirState.IN_MEMORY_MODIFIED
3008
self._mark_modified()
2862
3010
def _maybe_remove_row(self, block, index, id_index):
2863
3011
"""Remove index if it is absent or relocated across the row.
2865
3013
id_index is updated accordingly.
3014
:return: True if we removed the row, False otherwise
2867
3016
present_in_row = False
2868
3017
entry = block[index]
3144
3300
entry[1][0] = ('f', '', stat_value.st_size,
3145
3301
executable, DirState.NULLSTAT)
3302
worth_saving = False
3146
3303
elif minikind == 'd':
3147
3304
link_or_sha1 = None
3148
3305
entry[1][0] = ('d', '', 0, False, packed_stat)
3154
3311
state._get_block_entry_index(entry[0][0], entry[0][1], 0)
3155
3312
state._ensure_block(block_index, entry_index,
3156
3313
osutils.pathjoin(entry[0][0], entry[0][1]))
3315
worth_saving = False
3157
3316
elif minikind == 'l':
3317
if saved_minikind == 'l':
3318
worth_saving = False
3158
3319
link_or_sha1 = state._read_link(abspath, saved_link_or_sha1)
3159
3320
if state._cutoff_time is None:
3160
3321
state._sha_cutoff_time()
3166
3327
entry[1][0] = ('l', '', stat_value.st_size,
3167
3328
False, DirState.NULLSTAT)
3168
state._dirblock_state = DirState.IN_MEMORY_MODIFIED
3330
state._mark_modified([entry])
3169
3331
return link_or_sha1
added
removed
bzrlib/dirstate.py
