1
1
# Copyright (C) 2010 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
from io import BytesIO
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from __future__ import absolute_import
20
from .lazy_import import lazy_import
22
from ...lazy_import import lazy_import
21
23
lazy_import(globals(), """
22
24
from fnmatch import fnmatch
24
26
from breezy._termcolor import color_string, FG
26
28
from breezy import (
33
revision as _mod_revision,
34
revision as _mod_revision,
37
from .revisionspec import (
40
from breezy.revisionspec import (
39
42
RevisionSpec_revid,
40
43
RevisionSpec_revno,
45
from breezy.sixish import (
43
49
_user_encoding = osutils.get_user_encoding()
47
53
"""Raised when a revision is not on left-hand history."""
50
class GrepOptions(object):
51
"""Container to pass around grep options.
53
This class is used as a container to pass around user option and
54
some other params (like outf) to processing functions. This makes
55
it easier to add more options as grep evolves.
70
files_with_matches = False
71
files_without_match = False
86
56
def _rev_on_mainline(rev_tuple):
87
57
"""returns True is rev tuple is on mainline"""
88
58
if len(rev_tuple) == 1:
153
123
def compile_pattern(pattern, flags=0):
155
return re.compile(pattern, flags)
126
# use python's re.compile as we need to catch re.error in case of bad pattern
127
lazy_regex.reset_compile()
128
patternc = re.compile(pattern, flags)
156
129
except re.error as e:
157
130
raise errors.BzrError("Invalid pattern: '%s'" % pattern)
161
134
def is_fixed_string(s):
162
if re.match("^([A-Za-z0-9_]|\\s)*$", s):
135
if re.match("^([A-Za-z0-9_]|\s)*$", s):
173
146
self.outf = opts.outf
174
147
if opts.show_color:
148
pat = opts.pattern.encode(_user_encoding, 'replace')
175
149
if opts.fixed_string:
176
self._old = opts.pattern
177
self._new = color_string(opts.pattern, FG.BOLD_RED)
151
self._new = color_string(pat, FG.BOLD_RED)
178
152
self.get_writer = self._get_writer_fixed_highlighted
180
154
flags = opts.patternc.flags
181
self._sub = re.compile(
182
opts.pattern.join(("((?:", ")+)")), flags).sub
155
self._sub = re.compile(pat.join(("((?:", ")+)")), flags).sub
183
156
self._highlight = color_string("\\1", FG.BOLD_RED)
184
157
self.get_writer = self._get_writer_regexp_highlighted
230
198
"""Get function for writing output with regexp match highlighted"""
231
199
_line_writer = self._get_writer_plain()
232
200
sub, highlight = self._sub, self._highlight
234
201
def _line_writer_regexp_highlighted(line):
235
202
"""Write formatted line with matched pattern highlighted"""
236
203
return _line_writer(line=sub(highlight, line))
240
207
"""Get function for writing output with search string highlighted"""
241
208
_line_writer = self._get_writer_plain()
242
209
old, new = self._old, self._new
244
210
def _line_writer_fixed_highlighted(line):
245
211
"""Write formatted line with string searched for highlighted"""
246
212
return _line_writer(line=line.replace(old, new))
257
223
# if no revision is sepcified for diff grep we grep all changesets.
258
224
opts.revision = [RevisionSpec.from_string('revno:1'),
259
RevisionSpec.from_string('last:1')]
225
RevisionSpec.from_string('last:1')]
260
226
start_rev = opts.revision[0]
261
227
start_revid = start_rev.as_revision_id(branch)
262
if start_revid == b'null:':
228
if start_revid == 'null:':
264
230
srevno_tuple = branch.revision_id_to_dotted_revno(start_revid)
265
231
if len(opts.revision) == 2:
269
235
end_revno, end_revid = branch.last_revision_info()
270
236
erevno_tuple = branch.revision_id_to_dotted_revno(end_revid)
272
grep_mainline = (_rev_on_mainline(srevno_tuple)
273
and _rev_on_mainline(erevno_tuple))
238
grep_mainline = (_rev_on_mainline(srevno_tuple) and
239
_rev_on_mainline(erevno_tuple))
275
241
# ensure that we go in reverse order
276
242
if srevno_tuple > erevno_tuple:
281
247
# faster when we don't want to look at merged revs. We try this
282
248
# with _linear_view_revisions. If all revs are to be grepped we
283
249
# use the slower _graph_view_revisions
284
if opts.levels == 1 and grep_mainline:
285
given_revs = _linear_view_revisions(
286
branch, start_revid, end_revid)
250
if opts.levels==1 and grep_mainline:
251
given_revs = _linear_view_revisions(branch, start_revid, end_revid)
288
given_revs = _graph_view_revisions(
289
branch, start_revid, end_revid)
253
given_revs = _graph_view_revisions(branch, start_revid, end_revid)
291
255
# We do an optimization below. For grepping a specific revison
292
256
# We don't need to call _graph_view_revisions which is slow.
296
260
start_rev_tuple = (start_revid, start_revno, 0)
297
261
given_revs = [start_rev_tuple]
298
262
repo = branch.repository
299
diff_pattern = re.compile(
300
b"^[+\\-].*(" + opts.pattern.encode(_user_encoding) + b")")
301
file_pattern = re.compile(b"=== (modified|added|removed) file '.*'")
263
diff_pattern = re.compile("^[+\-].*(" + opts.pattern + ")")
264
file_pattern = re.compile("=== (modified|added|removed) file '.*'", re.UNICODE)
302
265
outputter = _GrepDiffOutputter(opts)
303
266
writeline = outputter.get_writer()
304
267
writerevno = outputter.get_revision_header_writer()
309
272
# with level=1 show only top level
312
rev_spec = RevisionSpec_revid.from_string(
313
"revid:" + revid.decode('utf-8'))
275
rev_spec = RevisionSpec_revid.from_string("revid:"+revid)
314
276
new_rev = repo.get_revision(revid)
315
277
new_tree = rev_spec.as_tree(branch)
316
278
if len(new_rev.parent_ids) == 0:
320
282
old_tree = repo.revision_tree(ancestor_id)
322
284
diff.show_diff_trees(old_tree, new_tree, s,
323
old_label='', new_label='')
285
old_label='', new_label='')
324
286
display_revno = True
325
287
display_file = False
326
288
file_header = None
334
296
writerevno("=== revno:%s ===" % (revno,))
335
297
display_revno = False
338
" %s" % (file_header.decode(file_encoding, 'replace'),))
299
writefileheader(" %s" % (file_header,))
339
300
display_file = False
340
301
line = line.decode(file_encoding, 'replace')
341
302
writeline(" %s" % (line,))
359
320
end_revno, end_revid = branch.last_revision_info()
360
321
erevno_tuple = branch.revision_id_to_dotted_revno(end_revid)
362
grep_mainline = (_rev_on_mainline(srevno_tuple)
363
and _rev_on_mainline(erevno_tuple))
323
grep_mainline = (_rev_on_mainline(srevno_tuple) and
324
_rev_on_mainline(erevno_tuple))
365
326
# ensure that we go in reverse order
366
327
if srevno_tuple > erevno_tuple:
372
333
# with _linear_view_revisions. If all revs are to be grepped we
373
334
# use the slower _graph_view_revisions
374
335
if opts.levels == 1 and grep_mainline:
375
given_revs = _linear_view_revisions(
376
branch, start_revid, end_revid)
336
given_revs = _linear_view_revisions(branch, start_revid, end_revid)
378
given_revs = _graph_view_revisions(
379
branch, start_revid, end_revid)
338
given_revs = _graph_view_revisions(branch, start_revid, end_revid)
381
340
# We do an optimization below. For grepping a specific revison
382
341
# We don't need to call _graph_view_revisions which is slow.
394
353
# with level=1 show only top level
397
rev = RevisionSpec_revid.from_string(
398
"revid:" + revid.decode('utf-8'))
356
rev = RevisionSpec_revid.from_string("revid:"+revid)
399
357
tree = rev.as_tree(branch)
400
358
for path in opts.path_list:
401
tree_path = osutils.pathjoin(relpath, path)
402
if not tree.has_filename(tree_path):
403
trace.warning("Skipped unknown file '%s'.", path)
359
path_for_id = osutils.pathjoin(relpath, path)
360
id = tree.path2id(path_for_id)
362
trace.warning("Skipped unknown file '%s'." % path)
406
365
if osutils.isdir(path):
407
366
path_prefix = path
408
367
dir_grep(tree, path, relpath, opts, revno, path_prefix)
411
tree, tree_path, '.', path, opts, revno)
369
versioned_file_grep(tree, id, '.', path, opts, revno)
414
372
def workingtree_grep(opts):
415
revno = opts.print_revno = None # for working tree set revno to None
373
revno = opts.print_revno = None # for working tree set revno to None
417
375
tree, branch, relpath = \
418
376
controldir.ControlDir.open_containing_tree_or_branch('.')
420
378
msg = ('Cannot search working tree. Working tree not found.\n'
421
'To search for specific revision in history use the -r option.')
422
raise errors.CommandError(msg)
379
'To search for specific revision in history use the -r option.')
380
raise errors.BzrCommandError(msg)
424
382
# GZ 2010-06-02: Shouldn't be smuggling this on opts, but easy for now
425
383
opts.outputter = _Outputter(opts)
430
388
path_prefix = path
431
389
dir_grep(tree, path, relpath, opts, revno, path_prefix)
433
with open(path, 'rb') as f:
434
_file_grep(f.read(), path, opts, revno)
391
_file_grep(open(path).read(), path, opts, revno)
437
394
def _skip_file(include, exclude, path):
460
417
# and hits manually refilled. Could do this again if it was
461
418
# for a good reason, otherwise cache might want purging.
462
419
outputter = opts.outputter
463
for fp, fc, fkind, entry in tree.list_files(
464
include_root=False, from_dir=from_dir, recursive=opts.recursive):
420
for fp, fc, fkind, fid, entry in tree.list_files(include_root=False,
421
from_dir=from_dir, recursive=opts.recursive):
466
423
if _skip_file(opts.include, opts.exclude, fp):
469
426
if fc == 'V' and fkind == 'file':
470
tree_path = osutils.pathjoin(from_dir if from_dir else '', fp)
471
if revno is not None:
472
428
# If old result is valid, print results immediately.
473
429
# Otherwise, add file info to to_grep so that the
474
430
# loop later will get chunks and grep them
475
cache_id = tree.get_file_revision(tree_path)
431
cache_id = tree.get_file_revision(fp, fid)
476
432
if cache_id in outputter.cache:
477
433
# GZ 2010-06-05: Not really sure caching and re-outputting
478
434
# the old path is really the right thing,
479
435
# but it's what the old code seemed to do
480
436
outputter.write_cached_lines(cache_id, revno)
482
to_grep_append((tree_path, (fp, tree_path)))
438
to_grep_append((fid, (fp, fid)))
484
440
# we are grepping working tree.
485
441
if from_dir is None:
489
445
if opts.files_with_matches or opts.files_without_match:
490
446
# Optimize for wtree list-only as we don't need to read the
492
with open(path_for_file, 'rb', buffering=4096) as file:
493
_file_grep_list_only_wtree(file, fp, opts, path_prefix)
448
file = open(path_for_file, 'r', buffering=4096)
449
_file_grep_list_only_wtree(file, fp, opts, path_prefix)
495
with open(path_for_file, 'rb') as f:
496
_file_grep(f.read(), fp, opts, revno, path_prefix)
451
file_text = open(path_for_file, 'r').read()
452
_file_grep(file_text, fp, opts, revno, path_prefix)
498
if revno is not None: # grep versioned files
499
for (path, tree_path), chunks in tree.iter_files_bytes(to_grep):
454
if revno != None: # grep versioned files
455
for (path, fid), chunks in tree.iter_files_bytes(to_grep):
500
456
path = _make_display_path(relpath, path)
501
_file_grep(b''.join(chunks), path, opts, revno, path_prefix,
502
tree.get_file_revision(tree_path))
457
_file_grep(chunks[0], path, opts, revno, path_prefix,
458
tree.get_file_revision(path, fid))
505
461
def _make_display_path(relpath, path):
520
def versioned_file_grep(tree, tree_path, relpath, path, opts, revno, path_prefix=None):
476
def versioned_file_grep(tree, id, relpath, path, opts, revno, path_prefix = None):
521
477
"""Create a file object for the specified id and pass it on to _file_grep.
524
480
path = _make_display_path(relpath, path)
525
file_text = tree.get_file_text(tree_path)
481
file_text = tree.get_file_text(relpath, id)
526
482
_file_grep(file_text, path, opts, revno, path_prefix)
536
492
def _file_grep_list_only_wtree(file, path, opts, path_prefix=None):
537
493
# test and skip binary files
538
if b'\x00' in file.read(1024):
494
if '\x00' in file.read(1024):
540
trace.warning("Binary file '%s' skipped.", path)
496
trace.warning("Binary file '%s' skipped." % path)
543
file.seek(0) # search from beginning
499
file.seek(0) # search from beginning
546
502
if opts.fixed_string:
549
505
if pattern in line:
552
else: # not fixed_string
508
else: # not fixed_string
553
509
for line in file:
554
510
if opts.patternc.search(line):
558
514
if (opts.files_with_matches and found) or \
559
(opts.files_without_match and not found):
515
(opts.files_without_match and not found):
560
516
if path_prefix and path_prefix != '.':
561
517
# user has passed a dir arg, show that as result prefix
562
518
path = osutils.pathjoin(path_prefix, path)
569
525
The idea here is to do this work only once per run, and finally return a
570
526
function that will do the minimum amount possible for each match.
573
528
def __init__(self, opts, use_cache=False):
574
529
self.outf = opts.outf
584
539
no_line = opts.files_with_matches or opts.files_without_match
586
541
if opts.show_color:
542
pat = opts.pattern.encode(_user_encoding, 'replace')
588
544
self.get_writer = self._get_writer_plain
589
545
elif opts.fixed_string:
590
self._old = opts.pattern
591
self._new = color_string(opts.pattern, FG.BOLD_RED)
547
self._new = color_string(pat, FG.BOLD_RED)
592
548
self.get_writer = self._get_writer_fixed_highlighted
594
550
flags = opts.patternc.flags
595
self._sub = re.compile(
596
opts.pattern.join(("((?:", ")+)")), flags).sub
551
self._sub = re.compile(pat.join(("((?:", ")+)")), flags).sub
597
552
self._highlight = color_string("\\1", FG.BOLD_RED)
598
553
self.get_writer = self._get_writer_regexp_highlighted
599
554
path_start = FG.MAGENTA
624
579
def _get_writer_plain(self, path, revno, cache_id):
625
580
"""Get function for writing uncoloured output"""
626
581
per_line = self._format_perline
627
start = self._format_initial % {"path": path, "revno": revno}
582
start = self._format_initial % {"path":path, "revno":revno}
628
583
write = self.outf.write
629
584
if self.cache is not None and cache_id is not None:
631
586
self.cache[cache_id] = path, result_list
632
587
add_to_cache = result_list.append
634
588
def _line_cache_and_writer(**kwargs):
635
589
"""Write formatted line and cache arguments"""
636
590
end = per_line % kwargs
637
591
add_to_cache(end)
638
592
write(start + end)
639
593
return _line_cache_and_writer
641
594
def _line_writer(**kwargs):
642
595
"""Write formatted line from arguments given by underlying opts"""
643
596
write(start + per_line % kwargs)
646
599
def write_cached_lines(self, cache_id, revno):
647
600
"""Write cached results out again for new revision"""
648
601
cached_path, cached_matches = self.cache[cache_id]
649
start = self._format_initial % {"path": cached_path, "revno": revno}
602
start = self._format_initial % {"path":cached_path, "revno":revno}
650
603
write = self.outf.write
651
604
for end in cached_matches:
652
605
write(start + end)
655
608
"""Get function for writing output with regexp match highlighted"""
656
609
_line_writer = self._get_writer_plain(path, revno, cache_id)
657
610
sub, highlight = self._sub, self._highlight
659
611
def _line_writer_regexp_highlighted(line, **kwargs):
660
612
"""Write formatted line with matched pattern highlighted"""
661
613
return _line_writer(line=sub(highlight, line), **kwargs)
665
617
"""Get function for writing output with search string highlighted"""
666
618
_line_writer = self._get_writer_plain(path, revno, cache_id)
667
619
old, new = self._old, self._new
669
620
def _line_writer_fixed_highlighted(line, **kwargs):
670
621
"""Write formatted line with string searched for highlighted"""
671
622
return _line_writer(line=line.replace(old, new), **kwargs)
675
626
def _file_grep(file_text, path, opts, revno, path_prefix=None, cache_id=None):
676
627
# test and skip binary files
677
if b'\x00' in file_text[:1024]:
628
if '\x00' in file_text[:1024]:
679
trace.warning("Binary file '%s' skipped.", path)
630
trace.warning("Binary file '%s' skipped." % path)
682
633
if path_prefix and path_prefix != '.':
714
665
i = file_text.find(pattern)
717
b = file_text.rfind(b"\n", 0, i) + 1
668
b = file_text.rfind("\n", 0, i) + 1
718
669
if opts.line_number:
719
start = file_text.count(b"\n", 0, b) + 1
670
start = file_text.count("\n", 0, b) + 1
720
671
file_text = file_text[b:]
721
672
if opts.line_number:
722
673
for index, line in enumerate(file_text.splitlines()):
723
674
if pattern in line:
724
675
line = line.decode(file_encoding, 'replace')
725
writeline(lineno=index + start, line=line)
676
writeline(lineno=index+start, line=line)
727
678
for line in file_text.splitlines():
728
679
if pattern in line:
733
684
# standard cases, but perhaps could try and detect backtracking
734
685
# patterns here and avoid whole text search in those cases
735
686
search = opts.patternc.search
736
if b"$" not in pattern:
687
if "$" not in pattern:
737
688
# GZ 2010-06-05: Grr, re.MULTILINE can't save us when searching
738
689
# through revisions as bazaar returns binary mode
739
690
# and trailing \r breaks $ as line ending match
740
691
m = search(file_text)
743
b = file_text.rfind(b"\n", 0, m.start()) + 1
694
b = file_text.rfind("\n", 0, m.start()) + 1
744
695
if opts.line_number:
745
start = file_text.count(b"\n", 0, b) + 1
696
start = file_text.count("\n", 0, b) + 1
746
697
file_text = file_text[b:]
750
701
for index, line in enumerate(file_text.splitlines()):
752
703
line = line.decode(file_encoding, 'replace')
753
writeline(lineno=index + start, line=line)
704
writeline(lineno=index+start, line=line)
755
706
for line in file_text.splitlines():
757
708
line = line.decode(file_encoding, 'replace')
758
709
writeline(line=line)
added
removed
breezy/plugins/grep/grep.py
