/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/dirstate.py

  • Committer: Robert Collins
  • Date: 2009-07-10 05:18:29 UTC
  • mto: This revision was merged to the branch mainline in revision 4524.
  • Revision ID: robertc@robertcollins.net-20090710051829-zesmf7tf2jwohfip
Fix fallout from the delta checking work, don't error on deltas containing the root inventory item in CHK delta application, and clean up Inventory docs.

Show diffs side-by-side

added added

removed removed

Lines of Context:
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2006, 2007, 2008 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
204
204
import bisect
205
205
import binascii
206
206
import errno
207
 
import operator
208
207
import os
209
208
from stat import S_IEXEC
210
209
import stat
1278
1277
    def update_by_delta(self, delta):
1279
1278
        """Apply an inventory delta to the dirstate for tree 0
1280
1279
 
1281
 
        This is the workhorse for apply_inventory_delta in dirstate based
1282
 
        trees.
1283
 
 
1284
1280
        :param delta: An inventory delta.  See Inventory.apply_delta for
1285
1281
            details.
1286
1282
        """
1287
1283
        self._read_dirblocks_if_needed()
1288
 
        encode = cache_utf8.encode
1289
1284
        insertions = {}
1290
1285
        removals = {}
1291
 
        # Accumulate parent references (path_utf8, id), to check for parentless
1292
 
        # items or items placed under files/links/tree-references. We get
1293
 
        # references from every item in the delta that is not a deletion and
1294
 
        # is not itself the root.
1295
 
        parents = set()
1296
 
        # Added ids must not be in the dirstate already. This set holds those
1297
 
        # ids.
1298
 
        new_ids = set()
1299
 
        # This loop transforms the delta to single atomic operations that can
1300
 
        # be executed and validated.
1301
 
        for old_path, new_path, file_id, inv_entry in sorted(
1302
 
            inventory._check_delta_unique_old_paths(
1303
 
            inventory._check_delta_unique_new_paths(
1304
 
            inventory._check_delta_ids_match_entry(
1305
 
            inventory._check_delta_ids_are_valid(
1306
 
            inventory._check_delta_new_path_entry_both_or_None(delta))))),
1307
 
            reverse=True):
 
1286
        for old_path, new_path, file_id, inv_entry in sorted(delta, reverse=True):
1308
1287
            if (file_id in insertions) or (file_id in removals):
1309
1288
                raise errors.InconsistentDelta(old_path or new_path, file_id,
1310
1289
                    "repeated file_id")
1311
1290
            if old_path is not None:
1312
1291
                old_path = old_path.encode('utf-8')
1313
1292
                removals[file_id] = old_path
1314
 
            else:
1315
 
                new_ids.add(file_id)
1316
1293
            if new_path is not None:
1317
 
                if inv_entry is None:
1318
 
                    raise errors.InconsistentDelta(new_path, file_id,
1319
 
                        "new_path with no entry")
1320
1294
                new_path = new_path.encode('utf-8')
1321
 
                dirname_utf8, basename = osutils.split(new_path)
1322
 
                if basename:
1323
 
                    parents.add((dirname_utf8, inv_entry.parent_id))
1324
 
                key = (dirname_utf8, basename, file_id)
 
1295
                dirname, basename = osutils.split(new_path)
 
1296
                key = (dirname, basename, file_id)
1325
1297
                minikind = DirState._kind_to_minikind[inv_entry.kind]
1326
1298
                if minikind == 't':
1327
 
                    fingerprint = inv_entry.reference_revision or ''
 
1299
                    fingerprint = inv_entry.reference_revision
1328
1300
                else:
1329
1301
                    fingerprint = ''
1330
1302
                insertions[file_id] = (key, minikind, inv_entry.executable,
1339
1311
                    minikind = child[1][0][0]
1340
1312
                    fingerprint = child[1][0][4]
1341
1313
                    executable = child[1][0][3]
1342
 
                    old_child_path = osutils.pathjoin(child_dirname,
1343
 
                                                      child_basename)
 
1314
                    old_child_path = osutils.pathjoin(child[0][0],
 
1315
                                                      child[0][1])
1344
1316
                    removals[child[0][2]] = old_child_path
1345
1317
                    child_suffix = child_dirname[len(old_path):]
1346
1318
                    new_child_dirname = (new_path + child_suffix)
1347
1319
                    key = (new_child_dirname, child_basename, child[0][2])
1348
 
                    new_child_path = osutils.pathjoin(new_child_dirname,
1349
 
                                                      child_basename)
 
1320
                    new_child_path = os.path.join(new_child_dirname,
 
1321
                                                  child_basename)
1350
1322
                    insertions[child[0][2]] = (key, minikind, executable,
1351
1323
                                               fingerprint, new_child_path)
1352
 
        self._check_delta_ids_absent(new_ids, delta, 0)
1353
 
        try:
1354
 
            self._apply_removals(removals.iteritems())
1355
 
            self._apply_insertions(insertions.values())
1356
 
            # Validate parents
1357
 
            self._after_delta_check_parents(parents, 0)
1358
 
        except errors.BzrError, e:
1359
 
            self._changes_aborted = True
1360
 
            if 'integrity error' not in str(e):
1361
 
                raise
1362
 
            # _get_entry raises BzrError when a request is inconsistent; we
1363
 
            # want such errors to be shown as InconsistentDelta - and that 
1364
 
            # fits the behaviour we trigger.
1365
 
            raise errors.InconsistentDeltaDelta(delta, "error from _get_entry.")
 
1324
        self._apply_removals(removals.values())
 
1325
        self._apply_insertions(insertions.values())
1366
1326
 
1367
1327
    def _apply_removals(self, removals):
1368
 
        for file_id, path in sorted(removals, reverse=True,
1369
 
            key=operator.itemgetter(1)):
 
1328
        for path in sorted(removals, reverse=True):
1370
1329
            dirname, basename = osutils.split(path)
1371
1330
            block_i, entry_i, d_present, f_present = \
1372
1331
                self._get_block_entry_index(dirname, basename, 0)
1373
 
            try:
1374
 
                entry = self._dirblocks[block_i][1][entry_i]
1375
 
            except IndexError:
1376
 
                self._changes_aborted = True
1377
 
                raise errors.InconsistentDelta(path, file_id,
1378
 
                    "Wrong path for old path.")
1379
 
            if not f_present or entry[1][0][0] in 'ar':
1380
 
                self._changes_aborted = True
1381
 
                raise errors.InconsistentDelta(path, file_id,
1382
 
                    "Wrong path for old path.")
1383
 
            if file_id != entry[0][2]:
1384
 
                self._changes_aborted = True
1385
 
                raise errors.InconsistentDelta(path, file_id,
1386
 
                    "Attempt to remove path has wrong id - found %r."
1387
 
                    % entry[0][2])
 
1332
            entry = self._dirblocks[block_i][1][entry_i]
1388
1333
            self._make_absent(entry)
1389
1334
            # See if we have a malformed delta: deleting a directory must not
1390
1335
            # leave crud behind. This increases the number of bisects needed
1398
1343
                # be due to it being in a parent tree, or a corrupt delta.
1399
1344
                for child_entry in self._dirblocks[block_i][1]:
1400
1345
                    if child_entry[1][0][0] not in ('r', 'a'):
1401
 
                        self._changes_aborted = True
1402
1346
                        raise errors.InconsistentDelta(path, entry[0][2],
1403
1347
                            "The file id was deleted but its children were "
1404
1348
                            "not deleted.")
1405
1349
 
1406
1350
    def _apply_insertions(self, adds):
1407
 
        try:
1408
 
            for key, minikind, executable, fingerprint, path_utf8 in sorted(adds):
1409
 
                self.update_minimal(key, minikind, executable, fingerprint,
1410
 
                                    path_utf8=path_utf8)
1411
 
        except errors.NotVersionedError:
1412
 
            self._changes_aborted = True
1413
 
            raise errors.InconsistentDelta(path_utf8.decode('utf8'), key[2],
1414
 
                "Missing parent")
 
1351
        for key, minikind, executable, fingerprint, path_utf8 in sorted(adds):
 
1352
            self.update_minimal(key, minikind, executable, fingerprint,
 
1353
                                path_utf8=path_utf8)
1415
1354
 
1416
1355
    def update_basis_by_delta(self, delta, new_revid):
1417
1356
        """Update the parents of this tree after a commit.
1461
1400
        # At the same time, to reduce interface friction we convert the input
1462
1401
        # inventory entries to dirstate.
1463
1402
        root_only = ('', '')
1464
 
        # Accumulate parent references (path_utf8, id), to check for parentless
1465
 
        # items or items placed under files/links/tree-references. We get
1466
 
        # references from every item in the delta that is not a deletion and
1467
 
        # is not itself the root.
 
1403
        # Accumulate parent references (path and id), to check for parentless
 
1404
        # items or items placed under files/links/tree-references.
1468
1405
        parents = set()
1469
 
        # Added ids must not be in the dirstate already. This set holds those
1470
 
        # ids.
1471
 
        new_ids = set()
1472
1406
        for old_path, new_path, file_id, inv_entry in delta:
1473
1407
            if inv_entry is not None and file_id != inv_entry.file_id:
1474
1408
                raise errors.InconsistentDelta(new_path, file_id,
1475
1409
                    "mismatched entry file_id %r" % inv_entry)
1476
1410
            if new_path is not None:
1477
 
                if inv_entry is None:
1478
 
                    raise errors.InconsistentDelta(new_path, file_id,
1479
 
                        "new_path with no entry")
1480
1411
                new_path_utf8 = encode(new_path)
1481
1412
                # note the parent for validation
1482
 
                dirname_utf8, basename_utf8 = osutils.split(new_path_utf8)
1483
 
                if basename_utf8:
1484
 
                    parents.add((dirname_utf8, inv_entry.parent_id))
 
1413
                dirname, basename = osutils.split(new_path_utf8)
 
1414
                if basename:
 
1415
                    parents.add((dirname, inv_entry.parent_id))
1485
1416
            if old_path is None:
1486
1417
                adds.append((None, encode(new_path), file_id,
1487
1418
                    inv_to_entry(inv_entry), True))
1488
 
                new_ids.add(file_id)
1489
1419
            elif new_path is None:
1490
1420
                deletes.append((encode(old_path), None, file_id, None, True))
1491
1421
            elif (old_path, new_path) != root_only:
1534
1464
                # of everything.
1535
1465
                changes.append((encode(old_path), encode(new_path), file_id,
1536
1466
                    inv_to_entry(inv_entry)))
1537
 
        self._check_delta_ids_absent(new_ids, delta, 1)
 
1467
 
1538
1468
        try:
1539
1469
            # Finish expunging deletes/first half of renames.
1540
1470
            self._update_basis_apply_deletes(deletes)
1543
1473
            # Apply in-situ changes.
1544
1474
            self._update_basis_apply_changes(changes)
1545
1475
            # Validate parents
1546
 
            self._after_delta_check_parents(parents, 1)
 
1476
            self._update_basis_check_parents(parents)
1547
1477
        except errors.BzrError, e:
1548
 
            self._changes_aborted = True
1549
1478
            if 'integrity error' not in str(e):
1550
1479
                raise
1551
1480
            # _get_entry raises BzrError when a request is inconsistent; we
1553
1482
            # fits the behaviour we trigger. Partof this is driven by dirstate
1554
1483
            # only supporting deltas that turn the basis into a closer fit to
1555
1484
            # the active tree.
 
1485
            self._changes_aborted = True
1556
1486
            raise errors.InconsistentDeltaDelta(delta, "error from _get_entry.")
1557
1487
 
1558
1488
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
1560
1490
        self._id_index = None
1561
1491
        return
1562
1492
 
1563
 
    def _check_delta_ids_absent(self, new_ids, delta, tree_index):
1564
 
        """Check that none of the file_ids in new_ids are present in a tree."""
1565
 
        if not new_ids:
1566
 
            return
1567
 
        id_index = self._get_id_index()
1568
 
        for file_id in new_ids:
1569
 
            for key in id_index.get(file_id, []):
1570
 
                block_i, entry_i, d_present, f_present = \
1571
 
                    self._get_block_entry_index(key[0], key[1], tree_index)
1572
 
                if not f_present:
1573
 
                    # In a different tree
1574
 
                    continue
1575
 
                entry = self._dirblocks[block_i][1][entry_i]
1576
 
                if entry[0][2] != file_id:
1577
 
                    # Different file_id, so not what we want.
1578
 
                    continue
1579
 
                # NB: No changes made before this helper is called, so no need
1580
 
                # to set the _changes_aborted flag.
1581
 
                raise errors.InconsistentDelta(
1582
 
                    ("%s/%s" % key[0:2]).decode('utf8'), file_id,
1583
 
                    "This file_id is new in the delta but already present in "
1584
 
                    "the target")
1585
 
 
1586
1493
    def _update_basis_apply_adds(self, adds):
1587
1494
        """Apply a sequence of adds to tree 1 during update_basis_by_delta.
1588
1495
 
1653
1560
        null = DirState.NULL_PARENT_DETAILS
1654
1561
        for old_path, new_path, file_id, _, real_delete in deletes:
1655
1562
            if real_delete != (new_path is None):
1656
 
                self._changes_aborted = True
1657
1563
                raise AssertionError("bad delete delta")
1658
1564
            # the entry for this file_id must be in tree 1.
1659
1565
            dirname, basename = osutils.split(old_path)
1692
1598
                    # it is being resurrected here, so blank it out temporarily.
1693
1599
                    self._dirblocks[block_index][1][entry_index][1][1] = null
1694
1600
 
1695
 
    def _after_delta_check_parents(self, parents, index):
1696
 
        """Check that parents required by the delta are all intact.
1697
 
        
1698
 
        :param parents: An iterable of (path_utf8, file_id) tuples which are
1699
 
            required to be present in tree 'index' at path_utf8 with id file_id
1700
 
            and be a directory.
1701
 
        :param index: The column in the dirstate to check for parents in.
1702
 
        """
1703
 
        for dirname_utf8, file_id in parents:
1704
 
            # Get the entry - the ensures that file_id, dirname_utf8 exists and
1705
 
            # has the right file id.
1706
 
            entry = self._get_entry(index, file_id, dirname_utf8)
 
1601
    def _update_basis_check_parents(self, parents):
 
1602
        """Check that parents required by the delta are all intact."""
 
1603
        for dirname, file_id in parents:
 
1604
            # Get the entry - the ensures that file_id, dirname exists and has
 
1605
            # the right file id.
 
1606
            entry = self._get_entry(1, file_id, dirname)
1707
1607
            if entry[1] is None:
1708
1608
                self._changes_aborted = True
1709
 
                raise errors.InconsistentDelta(dirname_utf8.decode('utf8'),
1710
 
                    file_id, "This parent is not present.")
 
1609
                raise errors.InconsistentDelta(dirname.decode('utf8'), file_id,
 
1610
                    "This parent is not present.")
1711
1611
            # Parents of things must be directories
1712
 
            if entry[1][index][0] != 'd':
 
1612
            if entry[1][1][0] != 'd':
1713
1613
                self._changes_aborted = True
1714
 
                raise errors.InconsistentDelta(dirname_utf8.decode('utf8'),
1715
 
                    file_id, "This parent is not a directory.")
 
1614
                raise errors.InconsistentDelta(dirname.decode('utf8'), file_id,
 
1615
                    "This parent is not a directory.")
1716
1616
 
1717
1617
    def _observed_sha1(self, entry, sha1, stat_value,
1718
1618
        _stat_to_minikind=_stat_to_minikind, _pack_stat=pack_stat):
1997
1897
                entry_index, present = self._find_entry_index(key, block)
1998
1898
                if present:
1999
1899
                    entry = self._dirblocks[block_index][1][entry_index]
2000
 
                    # TODO: We might want to assert that entry[0][2] ==
2001
 
                    #       fileid_utf8.
2002
1900
                    if entry[1][tree_index][0] in 'fdlt':
2003
1901
                        # this is the result we are looking for: the
2004
1902
                        # real home of this file_id in this tree.
2356
2254
        self.update_minimal(('', '', new_id), 'd',
2357
2255
            path_utf8='', packed_stat=entry[1][0][4])
2358
2256
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
 
2257
        if self._id_index is not None:
 
2258
            self._id_index.setdefault(new_id, set()).add(entry[0])
2359
2259
 
2360
2260
    def set_parent_trees(self, trees, ghosts):
2361
2261
        """Set the parent trees for the dirstate.
2520
2420
        if 'evil' in debug.debug_flags:
2521
2421
            trace.mutter_callsite(1,
2522
2422
                "set_state_from_inventory called; please mutate the tree instead")
2523
 
        tracing = 'dirstate' in debug.debug_flags
2524
 
        if tracing:
2525
 
            trace.mutter("set_state_from_inventory trace:")
2526
2423
        self._read_dirblocks_if_needed()
2527
2424
        # sketch:
2528
2425
        # Two iterators: current data and new data, both in dirblock order.
2537
2434
        new_iterator = new_inv.iter_entries_by_dir()
2538
2435
        # we will be modifying the dirstate, so we need a stable iterator. In
2539
2436
        # future we might write one, for now we just clone the state into a
2540
 
        # list using a copy so that we see every original item and don't have
2541
 
        # to adjust the position when items are inserted or deleted in the
2542
 
        # underlying dirstate.
 
2437
        # list - which is a shallow copy.
2543
2438
        old_iterator = iter(list(self._iter_entries()))
2544
2439
        # both must have roots so this is safe:
2545
2440
        current_new = new_iterator.next()
2579
2474
            # we make both end conditions explicit
2580
2475
            if not current_old:
2581
2476
                # old is finished: insert current_new into the state.
2582
 
                if tracing:
2583
 
                    trace.mutter("Appending from new '%s'.",
2584
 
                        new_path_utf8.decode('utf8'))
2585
2477
                self.update_minimal(new_entry_key, current_new_minikind,
2586
2478
                    executable=current_new[1].executable,
2587
 
                    path_utf8=new_path_utf8, fingerprint=fingerprint,
2588
 
                    fullscan=True)
 
2479
                    path_utf8=new_path_utf8, fingerprint=fingerprint)
2589
2480
                current_new = advance(new_iterator)
2590
2481
            elif not current_new:
2591
2482
                # new is finished
2592
 
                if tracing:
2593
 
                    trace.mutter("Truncating from old '%s/%s'.",
2594
 
                        current_old[0][0].decode('utf8'),
2595
 
                        current_old[0][1].decode('utf8'))
2596
2483
                self._make_absent(current_old)
2597
2484
                current_old = advance(old_iterator)
2598
2485
            elif new_entry_key == current_old[0]:
2605
2492
                # kind has changed.
2606
2493
                if (current_old[1][0][3] != current_new[1].executable or
2607
2494
                    current_old[1][0][0] != current_new_minikind):
2608
 
                    if tracing:
2609
 
                        trace.mutter("Updating in-place change '%s'.",
2610
 
                            new_path_utf8.decode('utf8'))
2611
2495
                    self.update_minimal(current_old[0], current_new_minikind,
2612
2496
                        executable=current_new[1].executable,
2613
 
                        path_utf8=new_path_utf8, fingerprint=fingerprint,
2614
 
                        fullscan=True)
 
2497
                        path_utf8=new_path_utf8, fingerprint=fingerprint)
2615
2498
                # both sides are dealt with, move on
2616
2499
                current_old = advance(old_iterator)
2617
2500
                current_new = advance(new_iterator)
2620
2503
                      and new_entry_key[1:] < current_old[0][1:])):
2621
2504
                # new comes before:
2622
2505
                # add a entry for this and advance new
2623
 
                if tracing:
2624
 
                    trace.mutter("Inserting from new '%s'.",
2625
 
                        new_path_utf8.decode('utf8'))
2626
2506
                self.update_minimal(new_entry_key, current_new_minikind,
2627
2507
                    executable=current_new[1].executable,
2628
 
                    path_utf8=new_path_utf8, fingerprint=fingerprint,
2629
 
                    fullscan=True)
 
2508
                    path_utf8=new_path_utf8, fingerprint=fingerprint)
2630
2509
                current_new = advance(new_iterator)
2631
2510
            else:
2632
2511
                # we've advanced past the place where the old key would be,
2633
2512
                # without seeing it in the new list.  so it must be gone.
2634
 
                if tracing:
2635
 
                    trace.mutter("Deleting from old '%s/%s'.",
2636
 
                        current_old[0][0].decode('utf8'),
2637
 
                        current_old[0][1].decode('utf8'))
2638
2513
                self._make_absent(current_old)
2639
2514
                current_old = advance(old_iterator)
2640
2515
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
2641
2516
        self._id_index = None
2642
2517
        self._packed_stat_index = None
2643
 
        if tracing:
2644
 
            trace.mutter("set_state_from_inventory complete.")
2645
2518
 
2646
2519
    def _make_absent(self, current_old):
2647
2520
        """Mark current_old - an entry - as absent for tree 0.
2696
2569
        return last_reference
2697
2570
 
2698
2571
    def update_minimal(self, key, minikind, executable=False, fingerprint='',
2699
 
        packed_stat=None, size=0, path_utf8=None, fullscan=False):
 
2572
                       packed_stat=None, size=0, path_utf8=None):
2700
2573
        """Update an entry to the state in tree 0.
2701
2574
 
2702
2575
        This will either create a new entry at 'key' or update an existing one.
2713
2586
        :param size: Size information for new entry
2714
2587
        :param path_utf8: key[0] + '/' + key[1], just passed in to avoid doing
2715
2588
                extra computation.
2716
 
        :param fullscan: If True then a complete scan of the dirstate is being
2717
 
            done and checking for duplicate rows should not be done. This
2718
 
            should only be set by set_state_from_inventory and similar methods.
2719
2589
 
2720
2590
        If packed_stat and fingerprint are not given, they're invalidated in
2721
2591
        the entry.
2730
2600
        new_details = (minikind, fingerprint, size, executable, packed_stat)
2731
2601
        id_index = self._get_id_index()
2732
2602
        if not present:
2733
 
            # New record. Check there isn't a entry at this path already.
2734
 
            if not fullscan:
2735
 
                low_index, _ = self._find_entry_index(key[0:2] + ('',), block)
2736
 
                while low_index < len(block):
2737
 
                    entry = block[low_index]
2738
 
                    if entry[0][0:2] == key[0:2]:
2739
 
                        if entry[1][0][0] not in 'ar':
2740
 
                            # This entry has the same path (but a different id) as
2741
 
                            # the new entry we're adding, and is present in ths
2742
 
                            # tree.
2743
 
                            raise errors.InconsistentDelta(
2744
 
                                ("%s/%s" % key[0:2]).decode('utf8'), key[2],
2745
 
                                "Attempt to add item at path already occupied by "
2746
 
                                "id %r" % entry[0][2])
2747
 
                        low_index += 1
2748
 
                    else:
2749
 
                        break
2750
2603
            # new entry, synthesis cross reference here,
2751
2604
            existing_keys = id_index.setdefault(key[2], set())
2752
2605
            if not existing_keys:
2757
2610
                # grab one of them and use it to generate parent
2758
2611
                # relocation/absent entries.
2759
2612
                new_entry = key, [new_details]
2760
 
                # existing_keys can be changed as we iterate.
2761
 
                for other_key in tuple(existing_keys):
 
2613
                for other_key in existing_keys:
2762
2614
                    # change the record at other to be a pointer to this new
2763
2615
                    # record. The loop looks similar to the change to
2764
2616
                    # relocations when updating an existing record but its not:
2765
2617
                    # the test for existing kinds is different: this can be
2766
2618
                    # factored out to a helper though.
2767
 
                    other_block_index, present = self._find_block_index_from_key(
2768
 
                        other_key)
2769
 
                    if not present:
2770
 
                        raise AssertionError('could not find block for %s' % (
2771
 
                            other_key,))
2772
 
                    other_block = self._dirblocks[other_block_index][1]
2773
 
                    other_entry_index, present = self._find_entry_index(
2774
 
                        other_key, other_block)
2775
 
                    if not present:
2776
 
                        raise AssertionError(
2777
 
                            'update_minimal: could not find other entry for %s'
2778
 
                            % (other_key,))
 
2619
                    other_block_index, present = self._find_block_index_from_key(other_key)
 
2620
                    if not present:
 
2621
                        raise AssertionError('could not find block for %s' % (other_key,))
 
2622
                    other_entry_index, present = self._find_entry_index(other_key,
 
2623
                                            self._dirblocks[other_block_index][1])
 
2624
                    if not present:
 
2625
                        raise AssertionError('could not find entry for %s' % (other_key,))
2779
2626
                    if path_utf8 is None:
2780
2627
                        raise AssertionError('no path')
2781
 
                    # Turn this other location into a reference to the new
2782
 
                    # location. This also updates the aliased iterator
2783
 
                    # (current_old in set_state_from_inventory) so that the old
2784
 
                    # entry, if not already examined, is skipped over by that
2785
 
                    # loop.
2786
 
                    other_entry = other_block[other_entry_index]
2787
 
                    other_entry[1][0] = ('r', path_utf8, 0, False, '')
2788
 
                    self._maybe_remove_row(other_block, other_entry_index,
2789
 
                        id_index)
 
2628
                    self._dirblocks[other_block_index][1][other_entry_index][1][0] = \
 
2629
                        ('r', path_utf8, 0, False, '')
2790
2630
 
2791
 
                # This loop:
2792
 
                # adds a tuple to the new details for each column
2793
 
                #  - either by copying an existing relocation pointer inside that column
2794
 
                #  - or by creating a new pointer to the right row inside that column
2795
2631
                num_present_parents = self._num_present_parents()
2796
 
                if num_present_parents:
2797
 
                    other_key = list(existing_keys)[0]
2798
2632
                for lookup_index in xrange(1, num_present_parents + 1):
2799
2633
                    # grab any one entry, use it to find the right path.
2800
2634
                    # TODO: optimise this to reduce memory use in highly
2807
2641
                    update_entry_index, present = \
2808
2642
                        self._find_entry_index(other_key, self._dirblocks[update_block_index][1])
2809
2643
                    if not present:
2810
 
                        raise AssertionError('update_minimal: could not find entry for %s' % (other_key,))
 
2644
                        raise AssertionError('could not find entry for %s' % (other_key,))
2811
2645
                    update_details = self._dirblocks[update_block_index][1][update_entry_index][1][lookup_index]
2812
2646
                    if update_details[0] in 'ar': # relocated, absent
2813
2647
                        # its a pointer or absent in lookup_index's tree, use
2859
2693
 
2860
2694
        self._dirblock_state = DirState.IN_MEMORY_MODIFIED
2861
2695
 
2862
 
    def _maybe_remove_row(self, block, index, id_index):
2863
 
        """Remove index if it is absent or relocated across the row.
2864
 
        
2865
 
        id_index is updated accordingly.
2866
 
        """
2867
 
        present_in_row = False
2868
 
        entry = block[index]
2869
 
        for column in entry[1]:
2870
 
            if column[0] not in 'ar':
2871
 
                present_in_row = True
2872
 
                break
2873
 
        if not present_in_row:
2874
 
            block.pop(index)
2875
 
            id_index[entry[0][2]].remove(entry[0])
2876
 
 
2877
2696
    def _validate(self):
2878
2697
        """Check that invariants on the dirblock are correct.
2879
2698
 
3013
2832
            if absent_positions == tree_count:
3014
2833
                raise AssertionError(
3015
2834
                    "entry %r has no data for any tree." % (entry,))
3016
 
        if self._id_index is not None:
3017
 
            for file_id, entry_keys in self._id_index.iteritems():
3018
 
                for entry_key in entry_keys:
3019
 
                    if entry_key[2] != file_id:
3020
 
                        raise AssertionError(
3021
 
                            'file_id %r did not match entry key %s'
3022
 
                            % (file_id, entry_key))
3023
2835
 
3024
2836
    def _wipe_state(self):
3025
2837
        """Forget all state information about the dirstate."""
3171
2983
 
3172
2984
class ProcessEntryPython(object):
3173
2985
 
3174
 
    __slots__ = ["old_dirname_to_file_id", "new_dirname_to_file_id",
 
2986
    __slots__ = ["old_dirname_to_file_id", "new_dirname_to_file_id", "uninteresting",
3175
2987
        "last_source_parent", "last_target_parent", "include_unchanged",
3176
 
        "partial", "use_filesystem_for_exec", "utf8_decode",
3177
 
        "searched_specific_files", "search_specific_files",
3178
 
        "searched_exact_paths", "search_specific_file_parents", "seen_ids",
3179
 
        "state", "source_index", "target_index", "want_unversioned", "tree"]
 
2988
        "use_filesystem_for_exec", "utf8_decode", "searched_specific_files",
 
2989
        "search_specific_files", "state", "source_index", "target_index",
 
2990
        "want_unversioned", "tree"]
3180
2991
 
3181
2992
    def __init__(self, include_unchanged, use_filesystem_for_exec,
3182
2993
        search_specific_files, state, source_index, target_index,
3183
2994
        want_unversioned, tree):
3184
2995
        self.old_dirname_to_file_id = {}
3185
2996
        self.new_dirname_to_file_id = {}
3186
 
        # Are we doing a partial iter_changes?
3187
 
        self.partial = search_specific_files != set([''])
 
2997
        # Just a sentry, so that _process_entry can say that this
 
2998
        # record is handled, but isn't interesting to process (unchanged)
 
2999
        self.uninteresting = object()
3188
3000
        # Using a list so that we can access the values and change them in
3189
3001
        # nested scope. Each one is [path, file_id, entry]
3190
3002
        self.last_source_parent = [None, None]
3193
3005
        self.use_filesystem_for_exec = use_filesystem_for_exec
3194
3006
        self.utf8_decode = cache_utf8._utf8_decode
3195
3007
        # for all search_indexs in each path at or under each element of
3196
 
        # search_specific_files, if the detail is relocated: add the id, and
3197
 
        # add the relocated path as one to search if its not searched already.
3198
 
        # If the detail is not relocated, add the id.
 
3008
        # search_specific_files, if the detail is relocated: add the id, and add the
 
3009
        # relocated path as one to search if its not searched already. If the
 
3010
        # detail is not relocated, add the id.
3199
3011
        self.searched_specific_files = set()
3200
 
        # When we search exact paths without expanding downwards, we record
3201
 
        # that here.
3202
 
        self.searched_exact_paths = set()
3203
3012
        self.search_specific_files = search_specific_files
3204
 
        # The parents up to the root of the paths we are searching.
3205
 
        # After all normal paths are returned, these specific items are returned.
3206
 
        self.search_specific_file_parents = set()
3207
 
        # The ids we've sent out in the delta.
3208
 
        self.seen_ids = set()
3209
3013
        self.state = state
3210
3014
        self.source_index = source_index
3211
3015
        self.target_index = target_index
3212
 
        if target_index != 0:
3213
 
            # A lot of code in here depends on target_index == 0
3214
 
            raise errors.BzrError('unsupported target index')
3215
3016
        self.want_unversioned = want_unversioned
3216
3017
        self.tree = tree
3217
3018
 
3219
3020
        """Compare an entry and real disk to generate delta information.
3220
3021
 
3221
3022
        :param path_info: top_relpath, basename, kind, lstat, abspath for
3222
 
            the path of entry. If None, then the path is considered absent in 
3223
 
            the target (Perhaps we should pass in a concrete entry for this ?)
 
3023
            the path of entry. If None, then the path is considered absent.
 
3024
            (Perhaps we should pass in a concrete entry for this ?)
3224
3025
            Basename is returned as a utf8 string because we expect this
3225
3026
            tuple will be ignored, and don't want to take the time to
3226
3027
            decode.
3227
 
        :return: (iter_changes_result, changed). If the entry has not been
3228
 
            handled then changed is None. Otherwise it is False if no content
3229
 
            or metadata changes have occurred, and True if any content or
3230
 
            metadata change has occurred. If self.include_unchanged is True then
3231
 
            if changed is not None, iter_changes_result will always be a result
3232
 
            tuple. Otherwise, iter_changes_result is None unless changed is
3233
 
            True.
 
3028
        :return: None if these don't match
 
3029
                 A tuple of information about the change, or
 
3030
                 the object 'uninteresting' if these match, but are
 
3031
                 basically identical.
3234
3032
        """
3235
3033
        if self.source_index is None:
3236
3034
            source_details = DirState.NULL_PARENT_DETAILS
3335
3133
                        content_change = False
3336
3134
                    target_exec = False
3337
3135
                else:
3338
 
                    if path is None:
3339
 
                        path = pathjoin(old_dirname, old_basename)
3340
 
                    raise errors.BadFileKindError(path, path_info[2])
 
3136
                    raise Exception, "unknown kind %s" % path_info[2]
3341
3137
            if source_minikind == 'd':
3342
3138
                if path is None:
3343
3139
                    old_path = path = pathjoin(old_dirname, old_basename)
3344
3140
                self.old_dirname_to_file_id[old_path] = file_id
3345
3141
            # parent id is the entry for the path in the target tree
3346
 
            if old_basename and old_dirname == self.last_source_parent[0]:
 
3142
            if old_dirname == self.last_source_parent[0]:
3347
3143
                source_parent_id = self.last_source_parent[1]
3348
3144
            else:
3349
3145
                try:
3359
3155
                    self.last_source_parent[0] = old_dirname
3360
3156
                    self.last_source_parent[1] = source_parent_id
3361
3157
            new_dirname = entry[0][0]
3362
 
            if entry[0][1] and new_dirname == self.last_target_parent[0]:
 
3158
            if new_dirname == self.last_target_parent[0]:
3363
3159
                target_parent_id = self.last_target_parent[1]
3364
3160
            else:
3365
3161
                try:
3382
3178
                    self.last_target_parent[1] = target_parent_id
3383
3179
 
3384
3180
            source_exec = source_details[3]
3385
 
            changed = (content_change
 
3181
            if (self.include_unchanged
 
3182
                or content_change
3386
3183
                or source_parent_id != target_parent_id
3387
3184
                or old_basename != entry[0][1]
3388
3185
                or source_exec != target_exec
3389
 
                )
3390
 
            if not changed and not self.include_unchanged:
3391
 
                return None, False
3392
 
            else:
 
3186
                ):
3393
3187
                if old_path is None:
3394
3188
                    old_path = path = pathjoin(old_dirname, old_basename)
3395
3189
                    old_path_u = self.utf8_decode(old_path)[0]
3408
3202
                       (source_parent_id, target_parent_id),
3409
3203
                       (self.utf8_decode(old_basename)[0], self.utf8_decode(entry[0][1])[0]),
3410
3204
                       (source_kind, target_kind),
3411
 
                       (source_exec, target_exec)), changed
 
3205
                       (source_exec, target_exec))
 
3206
            else:
 
3207
                return self.uninteresting
3412
3208
        elif source_minikind in 'a' and target_minikind in 'fdlt':
3413
3209
            # looks like a new file
3414
3210
            path = pathjoin(entry[0][0], entry[0][1])
3435
3231
                       (None, parent_id),
3436
3232
                       (None, self.utf8_decode(entry[0][1])[0]),
3437
3233
                       (None, path_info[2]),
3438
 
                       (None, target_exec)), True
 
3234
                       (None, target_exec))
3439
3235
            else:
3440
3236
                # Its a missing file, report it as such.
3441
3237
                return (entry[0][2],
3445
3241
                       (None, parent_id),
3446
3242
                       (None, self.utf8_decode(entry[0][1])[0]),
3447
3243
                       (None, None),
3448
 
                       (None, False)), True
 
3244
                       (None, False))
3449
3245
        elif source_minikind in 'fdlt' and target_minikind in 'a':
3450
3246
            # unversioned, possibly, or possibly not deleted: we dont care.
3451
3247
            # if its still on disk, *and* theres no other entry at this
3463
3259
                   (parent_id, None),
3464
3260
                   (self.utf8_decode(entry[0][1])[0], None),
3465
3261
                   (DirState._minikind_to_kind[source_minikind], None),
3466
 
                   (source_details[3], None)), True
 
3262
                   (source_details[3], None))
3467
3263
        elif source_minikind in 'fdlt' and target_minikind in 'r':
3468
3264
            # a rename; could be a true rename, or a rename inherited from
3469
3265
            # a renamed parent. TODO: handle this efficiently. Its not
3481
3277
                "source_minikind=%r, target_minikind=%r"
3482
3278
                % (source_minikind, target_minikind))
3483
3279
            ## import pdb;pdb.set_trace()
3484
 
        return None, None
 
3280
        return None
3485
3281
 
3486
3282
    def __iter__(self):
3487
3283
        return self
3488
3284
 
3489
 
    def _gather_result_for_consistency(self, result):
3490
 
        """Check a result we will yield to make sure we are consistent later.
3491
 
        
3492
 
        This gathers result's parents into a set to output later.
3493
 
 
3494
 
        :param result: A result tuple.
3495
 
        """
3496
 
        if not self.partial or not result[0]:
3497
 
            return
3498
 
        self.seen_ids.add(result[0])
3499
 
        new_path = result[1][1]
3500
 
        if new_path:
3501
 
            # Not the root and not a delete: queue up the parents of the path.
3502
 
            self.search_specific_file_parents.update(
3503
 
                osutils.parent_directories(new_path.encode('utf8')))
3504
 
            # Add the root directory which parent_directories does not
3505
 
            # provide.
3506
 
            self.search_specific_file_parents.add('')
3507
 
 
3508
3285
    def iter_changes(self):
3509
3286
        """Iterate over the changes."""
3510
3287
        utf8_decode = cache_utf8._utf8_decode
3511
3288
        _cmp_by_dirs = cmp_by_dirs
3512
3289
        _process_entry = self._process_entry
 
3290
        uninteresting = self.uninteresting
3513
3291
        search_specific_files = self.search_specific_files
3514
3292
        searched_specific_files = self.searched_specific_files
3515
3293
        splitpath = osutils.splitpath
3585
3363
                continue
3586
3364
            path_handled = False
3587
3365
            for entry in root_entries:
3588
 
                result, changed = _process_entry(entry, root_dir_info)
3589
 
                if changed is not None:
 
3366
                result = _process_entry(entry, root_dir_info)
 
3367
                if result is not None:
3590
3368
                    path_handled = True
3591
 
                    if changed:
3592
 
                        self._gather_result_for_consistency(result)
3593
 
                    if changed or self.include_unchanged:
 
3369
                    if result is not uninteresting:
3594
3370
                        yield result
3595
3371
            if self.want_unversioned and not path_handled and root_dir_info:
3596
3372
                new_executable = bool(
3706
3482
                        for current_entry in current_block[1]:
3707
3483
                            # entry referring to file not present on disk.
3708
3484
                            # advance the entry only, after processing.
3709
 
                            result, changed = _process_entry(current_entry, None)
3710
 
                            if changed is not None:
3711
 
                                if changed:
3712
 
                                    self._gather_result_for_consistency(result)
3713
 
                                if changed or self.include_unchanged:
 
3485
                            result = _process_entry(current_entry, None)
 
3486
                            if result is not None:
 
3487
                                if result is not uninteresting:
3714
3488
                                    yield result
3715
3489
                        block_index +=1
3716
3490
                        if (block_index < len(self.state._dirblocks) and
3746
3520
                        pass
3747
3521
                    elif current_path_info is None:
3748
3522
                        # no path is fine: the per entry code will handle it.
3749
 
                        result, changed = _process_entry(current_entry, current_path_info)
3750
 
                        if changed is not None:
3751
 
                            if changed:
3752
 
                                self._gather_result_for_consistency(result)
3753
 
                            if changed or self.include_unchanged:
 
3523
                        result = _process_entry(current_entry, current_path_info)
 
3524
                        if result is not None:
 
3525
                            if result is not uninteresting:
3754
3526
                                yield result
3755
3527
                    elif (current_entry[0][1] != current_path_info[1]
3756
3528
                          or current_entry[1][self.target_index][0] in 'ar'):
3769
3541
                        else:
3770
3542
                            # entry referring to file not present on disk.
3771
3543
                            # advance the entry only, after processing.
3772
 
                            result, changed = _process_entry(current_entry, None)
3773
 
                            if changed is not None:
3774
 
                                if changed:
3775
 
                                    self._gather_result_for_consistency(result)
3776
 
                                if changed or self.include_unchanged:
 
3544
                            result = _process_entry(current_entry, None)
 
3545
                            if result is not None:
 
3546
                                if result is not uninteresting:
3777
3547
                                    yield result
3778
3548
                            advance_path = False
3779
3549
                    else:
3780
 
                        result, changed = _process_entry(current_entry, current_path_info)
3781
 
                        if changed is not None:
 
3550
                        result = _process_entry(current_entry, current_path_info)
 
3551
                        if result is not None:
3782
3552
                            path_handled = True
3783
 
                            if changed:
3784
 
                                self._gather_result_for_consistency(result)
3785
 
                            if changed or self.include_unchanged:
 
3553
                            if result is not uninteresting:
3786
3554
                                yield result
3787
3555
                    if advance_entry and current_entry is not None:
3788
3556
                        entry_index += 1
3847
3615
                        current_dir_info = dir_iterator.next()
3848
3616
                    except StopIteration:
3849
3617
                        current_dir_info = None
3850
 
        for result in self._iter_specific_file_parents():
3851
 
            yield result
3852
 
 
3853
 
    def _iter_specific_file_parents(self):
3854
 
        """Iter over the specific file parents."""
3855
 
        while self.search_specific_file_parents:
3856
 
            # Process the parent directories for the paths we were iterating.
3857
 
            # Even in extremely large trees this should be modest, so currently
3858
 
            # no attempt is made to optimise.
3859
 
            path_utf8 = self.search_specific_file_parents.pop()
3860
 
            if osutils.is_inside_any(self.searched_specific_files, path_utf8):
3861
 
                # We've examined this path.
3862
 
                continue
3863
 
            if path_utf8 in self.searched_exact_paths:
3864
 
                # We've examined this path.
3865
 
                continue
3866
 
            path_entries = self.state._entries_for_path(path_utf8)
3867
 
            # We need either one or two entries. If the path in
3868
 
            # self.target_index has moved (so the entry in source_index is in
3869
 
            # 'ar') then we need to also look for the entry for this path in
3870
 
            # self.source_index, to output the appropriate delete-or-rename.
3871
 
            selected_entries = []
3872
 
            found_item = False
3873
 
            for candidate_entry in path_entries:
3874
 
                # Find entries present in target at this path:
3875
 
                if candidate_entry[1][self.target_index][0] not in 'ar':
3876
 
                    found_item = True
3877
 
                    selected_entries.append(candidate_entry)
3878
 
                # Find entries present in source at this path:
3879
 
                elif (self.source_index is not None and
3880
 
                    candidate_entry[1][self.source_index][0] not in 'ar'):
3881
 
                    found_item = True
3882
 
                    if candidate_entry[1][self.target_index][0] == 'a':
3883
 
                        # Deleted, emit it here.
3884
 
                        selected_entries.append(candidate_entry)
3885
 
                    else:
3886
 
                        # renamed, emit it when we process the directory it
3887
 
                        # ended up at.
3888
 
                        self.search_specific_file_parents.add(
3889
 
                            candidate_entry[1][self.target_index][1])
3890
 
            if not found_item:
3891
 
                raise AssertionError(
3892
 
                    "Missing entry for specific path parent %r, %r" % (
3893
 
                    path_utf8, path_entries))
3894
 
            path_info = self._path_info(path_utf8, path_utf8.decode('utf8'))
3895
 
            for entry in selected_entries:
3896
 
                if entry[0][2] in self.seen_ids:
3897
 
                    continue
3898
 
                result, changed = self._process_entry(entry, path_info)
3899
 
                if changed is None:
3900
 
                    raise AssertionError(
3901
 
                        "Got entry<->path mismatch for specific path "
3902
 
                        "%r entry %r path_info %r " % (
3903
 
                        path_utf8, entry, path_info))
3904
 
                # Only include changes - we're outside the users requested
3905
 
                # expansion.
3906
 
                if changed:
3907
 
                    self._gather_result_for_consistency(result)
3908
 
                    if (result[6][0] == 'directory' and
3909
 
                        result[6][1] != 'directory'):
3910
 
                        # This stopped being a directory, the old children have
3911
 
                        # to be included.
3912
 
                        if entry[1][self.source_index][0] == 'r':
3913
 
                            # renamed, take the source path
3914
 
                            entry_path_utf8 = entry[1][self.source_index][1]
3915
 
                        else:
3916
 
                            entry_path_utf8 = path_utf8
3917
 
                        initial_key = (entry_path_utf8, '', '')
3918
 
                        block_index, _ = self.state._find_block_index_from_key(
3919
 
                            initial_key)
3920
 
                        if block_index == 0:
3921
 
                            # The children of the root are in block index 1.
3922
 
                            block_index +=1
3923
 
                        current_block = None
3924
 
                        if block_index < len(self.state._dirblocks):
3925
 
                            current_block = self.state._dirblocks[block_index]
3926
 
                            if not osutils.is_inside(
3927
 
                                entry_path_utf8, current_block[0]):
3928
 
                                # No entries for this directory at all.
3929
 
                                current_block = None
3930
 
                        if current_block is not None:
3931
 
                            for entry in current_block[1]:
3932
 
                                if entry[1][self.source_index][0] in 'ar':
3933
 
                                    # Not in the source tree, so doesn't have to be
3934
 
                                    # included.
3935
 
                                    continue
3936
 
                                # Path of the entry itself.
3937
 
 
3938
 
                                self.search_specific_file_parents.add(
3939
 
                                    osutils.pathjoin(*entry[0][:2]))
3940
 
                if changed or self.include_unchanged:
3941
 
                    yield result
3942
 
            self.searched_exact_paths.add(path_utf8)
3943
 
 
3944
 
    def _path_info(self, utf8_path, unicode_path):
3945
 
        """Generate path_info for unicode_path.
3946
 
 
3947
 
        :return: None if unicode_path does not exist, or a path_info tuple.
3948
 
        """
3949
 
        abspath = self.tree.abspath(unicode_path)
3950
 
        try:
3951
 
            stat = os.lstat(abspath)
3952
 
        except OSError, e:
3953
 
            if e.errno == errno.ENOENT:
3954
 
                # the path does not exist.
3955
 
                return None
3956
 
            else:
3957
 
                raise
3958
 
        utf8_basename = utf8_path.rsplit('/', 1)[-1]
3959
 
        dir_info = (utf8_path, utf8_basename,
3960
 
            osutils.file_kind_from_stat_mode(stat.st_mode), stat,
3961
 
            abspath)
3962
 
        if dir_info[2] == 'directory':
3963
 
            if self.tree._directory_is_tree_reference(
3964
 
                unicode_path):
3965
 
                self.root_dir_info = self.root_dir_info[:2] + \
3966
 
                    ('tree-reference',) + self.root_dir_info[3:]
3967
 
        return dir_info
3968
3618
 
3969
3619
 
3970
3620
# Try to load the compiled form if possible
3978
3628
        ProcessEntryC as _process_entry,
3979
3629
        update_entry as update_entry,
3980
3630
        )
3981
 
except ImportError, e:
3982
 
    osutils.failed_to_load_extension(e)
 
3631
except ImportError:
3983
3632
    from bzrlib._dirstate_helpers_py import (
3984
3633
        _read_dirblocks,
3985
3634
        bisect_dirblock,