1
# Copyright (C) 2007-2011 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
from __future__ import absolute_import
25
from .lazy_import import lazy_import
27
lazy_import(globals(), """
40
from .i18n import gettext
43
def topo_iter_keys(vf, keys=None):
46
parents = vf.get_parent_map(keys)
47
return _topo_iter(parents, keys)
50
def topo_iter(vf, versions=None):
52
versions = vf.versions()
53
parents = vf.get_parent_map(versions)
54
return _topo_iter(parents, versions)
57
def _topo_iter(parents, versions):
61
def pending_parents(version):
62
if parents[version] is None:
64
return [v for v in parents[version] if v in versions and
66
for version_id in versions:
67
if parents[version_id] is None:
70
for parent_id in parents[version_id]:
71
descendants.setdefault(parent_id, []).append(version_id)
72
cur = [v for v in versions if len(pending_parents(v)) == 0]
75
for version_id in cur:
76
if version_id in seen:
78
if len(pending_parents(version_id)) != 0:
80
next.extend(descendants.get(version_id, []))
86
class MultiParent(object):
87
"""A multi-parent diff"""
91
def __init__(self, hunks=None):
98
return "MultiParent(%r)" % self.hunks
100
def __eq__(self, other):
101
if self.__class__ is not other.__class__:
103
return (self.hunks == other.hunks)
106
def from_lines(text, parents=(), left_blocks=None):
107
"""Produce a MultiParent from a list of lines and parents"""
109
matcher = patiencediff.PatienceSequenceMatcher(None, parent,
111
return matcher.get_matching_blocks()
113
if left_blocks is None:
114
left_blocks = compare(parents[0])
115
parent_comparisons = [left_blocks] + [compare(p) for p in
118
parent_comparisons = []
120
new_text = NewText([])
122
block_iter = [iter(i) for i in parent_comparisons]
123
diff = MultiParent([])
127
return next(block_iter[p])
128
except StopIteration:
130
cur_block = [next_block(p) for p, i in enumerate(block_iter)]
131
while cur_line < len(text):
133
for p, block in enumerate(cur_block):
137
while j + n <= cur_line:
138
block = cur_block[p] = next_block(p)
146
offset = cur_line - j
152
if best_match is None or n > best_match.num_lines:
153
best_match = ParentText(p, i, j, n)
154
if best_match is None:
155
new_text.lines.append(text[cur_line])
158
if len(new_text.lines) > 0:
159
diff.hunks.append(new_text)
160
new_text = NewText([])
161
diff.hunks.append(best_match)
162
cur_line += best_match.num_lines
163
if len(new_text.lines) > 0:
164
diff.hunks.append(new_text)
167
def get_matching_blocks(self, parent, parent_len):
168
for hunk in self.hunks:
169
if not isinstance(hunk, ParentText) or hunk.parent != parent:
171
yield (hunk.parent_pos, hunk.child_pos, hunk.num_lines)
172
yield parent_len, self.num_lines(), 0
174
def to_lines(self, parents=()):
175
"""Contruct a fulltext from this diff and its parents"""
176
mpvf = MultiMemoryVersionedFile()
177
for num, parent in enumerate(parents):
178
mpvf.add_version(BytesIO(parent).readlines(), num, [])
179
mpvf.add_diff(self, 'a', list(range(len(parents))))
180
return mpvf.get_line_list(['a'])[0]
183
def from_texts(cls, text, parents=()):
184
"""Produce a MultiParent from a text and list of parent text"""
185
return cls.from_lines(BytesIO(text).readlines(),
186
[BytesIO(p).readlines() for p in parents])
189
"""Yield text lines for a patch"""
190
for hunk in self.hunks:
191
for line in hunk.to_patch():
195
return len(b''.join(self.to_patch()))
197
def zipped_patch_len(self):
198
return len(gzip_string(self.to_patch()))
201
def from_patch(cls, text):
202
"""Create a MultiParent from its string form"""
203
return cls._from_patch(BytesIO(text))
206
def _from_patch(lines):
207
"""This is private because it is essential to split lines on \n only"""
208
line_iter = iter(lines)
213
cur_line = next(line_iter)
214
except StopIteration:
216
first_char = cur_line[0:1]
217
if first_char == b'i':
218
num_lines = int(cur_line.split(b' ')[1])
219
hunk_lines = [next(line_iter) for _ in range(num_lines)]
220
hunk_lines[-1] = hunk_lines[-1][:-1]
221
hunks.append(NewText(hunk_lines))
222
elif first_char == b'\n':
223
hunks[-1].lines[-1] += b'\n'
225
if not (first_char == b'c'):
226
raise AssertionError(first_char)
227
parent, parent_pos, child_pos, num_lines =\
228
[int(v) for v in cur_line.split(b' ')[1:]]
229
hunks.append(ParentText(parent, parent_pos, child_pos,
231
return MultiParent(hunks)
233
def range_iterator(self):
234
"""Iterate through the hunks, with range indicated
236
kind is "new" or "parent".
237
for "new", data is a list of lines.
238
for "parent", data is (parent, parent_start, parent_end)
239
:return: a generator of (start, end, kind, data)
242
for hunk in self.hunks:
243
if isinstance(hunk, NewText):
245
end = start + len(hunk.lines)
249
start = hunk.child_pos
250
end = start + hunk.num_lines
251
data = (hunk.parent, hunk.parent_pos, hunk.parent_pos +
253
yield start, end, kind, data
257
"""The number of lines in the output text"""
259
for hunk in reversed(self.hunks):
260
if isinstance(hunk, ParentText):
261
return hunk.child_pos + hunk.num_lines + extra_n
262
extra_n += len(hunk.lines)
265
def is_snapshot(self):
266
"""Return true of this hunk is effectively a fulltext"""
267
if len(self.hunks) != 1:
269
return (isinstance(self.hunks[0], NewText))
272
class NewText(object):
273
"""The contents of text that is introduced by this text"""
275
__slots__ = ['lines']
277
def __init__(self, lines):
280
def __eq__(self, other):
281
if self.__class__ is not other.__class__:
283
return (other.lines == self.lines)
286
return 'NewText(%r)' % self.lines
289
yield b'i %d\n' % len(self.lines)
290
for line in self.lines:
295
class ParentText(object):
296
"""A reference to text present in a parent text"""
298
__slots__ = ['parent', 'parent_pos', 'child_pos', 'num_lines']
300
def __init__(self, parent, parent_pos, child_pos, num_lines):
302
self.parent_pos = parent_pos
303
self.child_pos = child_pos
304
self.num_lines = num_lines
307
return {b'parent': self.parent,
308
b'parent_pos': self.parent_pos,
309
b'child_pos': self.child_pos,
310
b'num_lines': self.num_lines}
313
return ('ParentText(%(parent)r, %(parent_pos)r, %(child_pos)r,'
314
' %(num_lines)r)' % self._as_dict())
316
def __eq__(self, other):
317
if self.__class__ is not other.__class__:
319
return self._as_dict() == other._as_dict()
322
yield (b'c %(parent)d %(parent_pos)d %(child_pos)d %(num_lines)d\n'
326
class BaseVersionedFile(object):
327
"""Pseudo-VersionedFile skeleton for MultiParent"""
329
def __init__(self, snapshot_interval=25, max_snapshots=None):
332
self._snapshots = set()
333
self.snapshot_interval = snapshot_interval
334
self.max_snapshots = max_snapshots
337
return iter(self._parents)
339
def has_version(self, version):
340
return version in self._parents
342
def do_snapshot(self, version_id, parent_ids):
343
"""Determine whether to perform a snapshot for this version"""
344
if self.snapshot_interval is None:
346
if self.max_snapshots is not None and\
347
len(self._snapshots) == self.max_snapshots:
349
if len(parent_ids) == 0:
351
for ignored in range(self.snapshot_interval):
352
if len(parent_ids) == 0:
354
version_ids = parent_ids
356
for version_id in version_ids:
357
if version_id not in self._snapshots:
358
parent_ids.extend(self._parents[version_id])
362
def add_version(self, lines, version_id, parent_ids,
363
force_snapshot=None, single_parent=False):
364
"""Add a version to the versionedfile
366
:param lines: The list of lines to add. Must be split on '\n'.
367
:param version_id: The version_id of the version to add
368
:param force_snapshot: If true, force this version to be added as a
369
snapshot version. If false, force this version to be added as a
370
diff. If none, determine this automatically.
371
:param single_parent: If true, use a single parent, rather than
374
if force_snapshot is None:
375
do_snapshot = self.do_snapshot(version_id, parent_ids)
377
do_snapshot = force_snapshot
379
self._snapshots.add(version_id)
380
diff = MultiParent([NewText(lines)])
383
parent_lines = self.get_line_list(parent_ids[:1])
385
parent_lines = self.get_line_list(parent_ids)
386
diff = MultiParent.from_lines(lines, parent_lines)
387
if diff.is_snapshot():
388
self._snapshots.add(version_id)
389
self.add_diff(diff, version_id, parent_ids)
390
self._lines[version_id] = lines
392
def get_parents(self, version_id):
393
return self._parents[version_id]
395
def make_snapshot(self, version_id):
396
snapdiff = MultiParent([NewText(self.cache_version(version_id))])
397
self.add_diff(snapdiff, version_id, self._parents[version_id])
398
self._snapshots.add(version_id)
400
def import_versionedfile(self, vf, snapshots, no_cache=True,
401
single_parent=False, verify=False):
402
"""Import all revisions of a versionedfile
404
:param vf: The versionedfile to import
405
:param snapshots: If provided, the revisions to make snapshots of.
406
Otherwise, this will be auto-determined
407
:param no_cache: If true, clear the cache after every add.
408
:param single_parent: If true, omit all but one parent text, (but
409
retain parent metadata).
411
if not (no_cache or not verify):
413
revisions = set(vf.versions())
414
total = len(revisions)
415
with ui.ui_factory.nested_progress_bar() as pb:
416
while len(revisions) > 0:
418
for revision in revisions:
419
parents = vf.get_parents(revision)
420
if [p for p in parents if p not in self._parents] != []:
422
lines = [a + b' ' + l for a, l in
423
vf.annotate(revision)]
424
if snapshots is None:
425
force_snapshot = None
427
force_snapshot = (revision in snapshots)
428
self.add_version(lines, revision, parents, force_snapshot,
435
if not (lines == self.get_line_list([revision])[0]):
436
raise AssertionError()
438
pb.update(gettext('Importing revisions'),
439
(total - len(revisions)) + len(added), total)
440
revisions = [r for r in revisions if r not in added]
442
def select_snapshots(self, vf):
443
"""Determine which versions to add as snapshots"""
446
for version_id in topo_iter(vf):
447
potential_build_ancestors = set(vf.get_parents(version_id))
448
parents = vf.get_parents(version_id)
449
if len(parents) == 0:
450
snapshots.add(version_id)
451
build_ancestors[version_id] = set()
453
for parent in vf.get_parents(version_id):
454
potential_build_ancestors.update(build_ancestors[parent])
455
if len(potential_build_ancestors) > self.snapshot_interval:
456
snapshots.add(version_id)
457
build_ancestors[version_id] = set()
459
build_ancestors[version_id] = potential_build_ancestors
462
def select_by_size(self, num):
463
"""Select snapshots for minimum output size"""
464
num -= len(self._snapshots)
465
new_snapshots = self.get_size_ranking()[-num:]
466
return [v for n, v in new_snapshots]
468
def get_size_ranking(self):
469
"""Get versions ranked by size"""
471
for version_id in self.versions():
472
if version_id in self._snapshots:
474
diff_len = self.get_diff(version_id).patch_len()
475
snapshot_len = MultiParent([NewText(
476
self.cache_version(version_id))]).patch_len()
477
versions.append((snapshot_len - diff_len, version_id))
481
def import_diffs(self, vf):
482
"""Import the diffs from another pseudo-versionedfile"""
483
for version_id in vf.versions():
484
self.add_diff(vf.get_diff(version_id), version_id,
485
vf._parents[version_id])
487
def get_build_ranking(self):
488
"""Return revisions sorted by how much they reduce build complexity"""
491
for version_id in topo_iter(self):
492
could_avoid[version_id] = set()
493
if version_id not in self._snapshots:
494
for parent_id in self._parents[version_id]:
495
could_avoid[version_id].update(could_avoid[parent_id])
496
could_avoid[version_id].update(self._parents)
497
could_avoid[version_id].discard(version_id)
498
for avoid_id in could_avoid[version_id]:
499
referenced_by.setdefault(avoid_id, set()).add(version_id)
500
available_versions = list(self.versions())
502
while len(available_versions) > 0:
503
available_versions.sort(key=lambda x:
504
len(could_avoid[x]) *
505
len(referenced_by.get(x, [])))
506
selected = available_versions.pop()
507
ranking.append(selected)
508
for version_id in referenced_by[selected]:
509
could_avoid[version_id].difference_update(
510
could_avoid[selected])
511
for version_id in could_avoid[selected]:
512
referenced_by[version_id].difference_update(
513
referenced_by[selected]
517
def clear_cache(self):
520
def get_line_list(self, version_ids):
521
return [self.cache_version(v) for v in version_ids]
523
def cache_version(self, version_id):
525
return self._lines[version_id]
528
diff = self.get_diff(version_id)
530
reconstructor = _Reconstructor(self, self._lines, self._parents)
531
reconstructor.reconstruct_version(lines, version_id)
532
self._lines[version_id] = lines
536
class MultiMemoryVersionedFile(BaseVersionedFile):
537
"""Memory-backed pseudo-versionedfile"""
539
def __init__(self, snapshot_interval=25, max_snapshots=None):
540
BaseVersionedFile.__init__(self, snapshot_interval, max_snapshots)
543
def add_diff(self, diff, version_id, parent_ids):
544
self._diffs[version_id] = diff
545
self._parents[version_id] = parent_ids
547
def get_diff(self, version_id):
549
return self._diffs[version_id]
551
raise errors.RevisionNotPresent(version_id, self)
557
class MultiVersionedFile(BaseVersionedFile):
558
"""Disk-backed pseudo-versionedfile"""
560
def __init__(self, filename, snapshot_interval=25, max_snapshots=None):
561
BaseVersionedFile.__init__(self, snapshot_interval, max_snapshots)
562
self._filename = filename
563
self._diff_offset = {}
565
def get_diff(self, version_id):
566
start, count = self._diff_offset[version_id]
567
with open(self._filename + '.mpknit', 'rb') as infile:
569
sio = BytesIO(infile.read(count))
570
with gzip.GzipFile(None, mode='rb', fileobj=sio) as zip_file:
571
file_version_id = zip_file.readline()
572
content = zip_file.read()
573
return MultiParent.from_patch(content)
575
def add_diff(self, diff, version_id, parent_ids):
576
with open(self._filename + '.mpknit', 'ab') as outfile:
577
outfile.seek(0, 2) # workaround for windows bug:
578
# .tell() for files opened in 'ab' mode
579
# before any write returns 0
580
start = outfile.tell()
581
with gzip.GzipFile(None, mode='ab', fileobj=outfile) as zipfile:
582
zipfile.writelines(itertools.chain(
583
[b'version %s\n' % version_id], diff.to_patch()))
585
self._diff_offset[version_id] = (start, end - start)
586
self._parents[version_id] = parent_ids
590
os.unlink(self._filename + '.mpknit')
592
if e.errno != errno.ENOENT:
595
os.unlink(self._filename + '.mpidx')
597
if e.errno != errno.ENOENT:
601
open(self._filename + '.mpidx', 'wb').write(bencode.bencode(
602
(self._parents, list(self._snapshots), self._diff_offset)))
605
self._parents, snapshots, self._diff_offset = bencode.bdecode(
606
open(self._filename + '.mpidx', 'rb').read())
607
self._snapshots = set(snapshots)
610
class _Reconstructor(object):
611
"""Build a text from the diffs, ancestry graph and cached lines"""
613
def __init__(self, diffs, lines, parents):
616
self.parents = parents
619
def reconstruct(self, lines, parent_text, version_id):
620
"""Append the lines referred to by a ParentText to lines"""
621
parent_id = self.parents[version_id][parent_text.parent]
622
end = parent_text.parent_pos + parent_text.num_lines
623
return self._reconstruct(lines, parent_id, parent_text.parent_pos,
626
def _reconstruct(self, lines, req_version_id, req_start, req_end):
627
"""Append lines for the requested version_id range"""
628
# stack of pending range requests
629
if req_start == req_end:
631
pending_reqs = [(req_version_id, req_start, req_end)]
632
while len(pending_reqs) > 0:
633
req_version_id, req_start, req_end = pending_reqs.pop()
634
# lazily allocate cursors for versions
635
if req_version_id in self.lines:
636
lines.extend(self.lines[req_version_id][req_start:req_end])
639
start, end, kind, data, iterator = self.cursor[req_version_id]
641
iterator = self.diffs.get_diff(req_version_id).range_iterator()
642
start, end, kind, data = next(iterator)
643
if start > req_start:
644
iterator = self.diffs.get_diff(req_version_id).range_iterator()
645
start, end, kind, data = next(iterator)
647
# find the first hunk relevant to the request
648
while end <= req_start:
649
start, end, kind, data = next(iterator)
650
self.cursor[req_version_id] = start, end, kind, data, iterator
651
# if the hunk can't satisfy the whole request, split it in two,
652
# and leave the second half for later.
654
pending_reqs.append((req_version_id, end, req_end))
657
lines.extend(data[req_start - start: (req_end - start)])
659
# If the hunk is a ParentText, rewrite it as a range request
660
# for the parent, and make it the next pending request.
661
parent, parent_start, parent_end = data
662
new_version_id = self.parents[req_version_id][parent]
663
new_start = parent_start + req_start - start
664
new_end = parent_end + req_end - end
665
pending_reqs.append((new_version_id, new_start, new_end))
667
def reconstruct_version(self, lines, version_id):
668
length = self.diffs.get_diff(version_id).num_lines()
669
return self._reconstruct(lines, version_id, 0, length)
672
def gzip_string(lines):
674
with gzip.GzipFile(None, mode='wb', fileobj=sio) as data_file:
675
data_file.writelines(lines)
676
return sio.getvalue()
added
removed
breezy/multiparent.py
