77
def recreate_search(self, repository, search_bytes, discard_excess=False):
78
"""Recreate a search from its serialised form.
80
:param discard_excess: If True, and the search refers to data we don't
81
have, just silently accept that fact - the verb calling
82
recreate_search trusts that clients will look for missing things
83
they expected and get it from elsewhere.
85
lines = search_bytes.split('\n')
86
if lines[0] == 'ancestry-of':
88
search_result = graph.PendingAncestryResult(heads, repository)
89
return search_result, None
90
elif lines[0] == 'search':
91
return self.recreate_search_from_recipe(repository, lines[1:],
92
discard_excess=discard_excess)
94
return (None, FailedSmartServerResponse(('BadSearch',)))
96
def recreate_search_from_recipe(self, repository, lines,
97
discard_excess=False):
98
"""Recreate a specific revision search (vs a from-tip search).
100
:param discard_excess: If True, and the search refers to data we don't
101
have, just silently accept that fact - the verb calling
102
recreate_search trusts that clients will look for missing things
103
they expected and get it from elsewhere.
73
def recreate_search(self, repository, recipe_bytes):
74
lines = recipe_bytes.split('\n')
105
75
start_keys = set(lines[0].split(' '))
106
76
exclude_keys = set(lines[1].split(' '))
107
77
revision_count = int(lines[2])
206
166
queried_revs.update(next_revs)
207
167
parent_map = repo_graph.get_parent_map(next_revs)
208
current_revs = next_revs
209
168
next_revs = set()
210
for revision_id in current_revs:
212
parents = parent_map.get(revision_id)
213
if parents is not None:
214
# adjust for the wire
215
if parents == (_mod_revision.NULL_REVISION,):
217
# prepare the next query
218
next_revs.update(parents)
219
encoded_id = revision_id
222
encoded_id = "missing:" + revision_id
224
if (revision_id not in client_seen_revs and
225
(not missing_rev or include_missing)):
169
for revision_id, parents in parent_map.iteritems():
170
# adjust for the wire
171
if parents == (_mod_revision.NULL_REVISION,):
173
# prepare the next query
174
next_revs.update(parents)
175
if revision_id not in client_seen_revs:
226
176
# Client does not have this revision, give it to it.
227
177
# add parents to the result
228
result[encoded_id] = parents
178
result[revision_id] = parents
229
179
# Approximate the serialized cost of this revision_id.
230
size_so_far += 2 + len(encoded_id) + sum(map(len, parents))
180
size_so_far += 2 + len(revision_id) + sum(map(len, parents))
231
181
# get all the directly asked for parents, and then flesh out to
232
182
# 64K (compressed) or so. We do one level of depth at a time to
233
183
# stay in sync with the client. The 250000 magic number is
287
237
return SuccessfulSmartServerResponse(('ok', ), '\n'.join(lines))
290
class SmartServerRepositoryGetRevIdForRevno(SmartServerRepositoryReadLocked):
292
def do_readlocked_repository_request(self, repository, revno,
294
"""Find the revid for a given revno, given a known revno/revid pair.
299
found_flag, result = repository.get_rev_id_for_revno(revno, known_pair)
300
except errors.RevisionNotPresent, err:
301
if err.revision_id != known_pair[1]:
302
raise AssertionError(
303
'get_rev_id_for_revno raised RevisionNotPresent for '
304
'non-initial revision: ' + err.revision_id)
305
return FailedSmartServerResponse(
306
('nosuchrevision', err.revision_id))
308
return SuccessfulSmartServerResponse(('ok', result))
310
earliest_revno, earliest_revid = result
311
return SuccessfulSmartServerResponse(
312
('history-incomplete', earliest_revno, earliest_revid))
315
240
class SmartServerRequestHasRevision(SmartServerRepositoryRequest):
317
242
def do_repository_request(self, repository, revision_id):
420
345
self._to_format = network_format_registry.get(to_network_name)
421
if self._should_fake_unknown():
422
return FailedSmartServerResponse(
423
('UnknownMethod', 'Repository.get_stream'))
424
346
return None # Signal that we want a body.
426
def _should_fake_unknown(self):
427
"""Return True if we should return UnknownMethod to the client.
429
This is a workaround for bugs in pre-1.19 clients that claim to
430
support receiving streams of CHK repositories. The pre-1.19 client
431
expects inventory records to be serialized in the format defined by
432
to_network_name, but in pre-1.19 (at least) that format definition
433
tries to use the xml5 serializer, which does not correctly handle
434
rich-roots. After 1.19 the client can also accept inventory-deltas
435
(which avoids this issue), and those clients will use the
436
Repository.get_stream_1.19 verb instead of this one.
437
So: if this repository is CHK, and the to_format doesn't match,
438
we should just fake an UnknownSmartMethod error so that the client
439
will fallback to VFS, rather than sending it a stream we know it
442
from_format = self._repository._format
443
to_format = self._to_format
444
if not from_format.supports_chks:
445
# Source not CHK: that's ok
447
if (to_format.supports_chks and
448
from_format.repository_class is to_format.repository_class and
449
from_format._serializer == to_format._serializer):
450
# Source is CHK, but target matches: that's ok
451
# (e.g. 2a->2a, or CHK2->2a)
453
# Source is CHK, and target is not CHK or incompatible CHK. We can't
454
# generate a compatible stream.
457
348
def do_body(self, body_bytes):
458
349
repository = self._repository
459
350
repository.lock_read()
461
search_result, error = self.recreate_search(repository, body_bytes,
352
search, error = self.recreate_search(repository, body_bytes)
463
353
if error is not None:
464
354
repository.unlock()
356
search = search.get_result()
466
357
source = repository._get_source(self._to_format)
467
stream = source.get_stream(search_result)
358
stream = source.get_stream(search)
468
359
except Exception:
469
360
exc_info = sys.exc_info()
519
399
yield pack_writer.end()
522
class _ByteStreamDecoder(object):
523
"""Helper for _byte_stream_to_stream.
525
The expected usage of this class is via the function _byte_stream_to_stream
526
which creates a _ByteStreamDecoder, pops off the stream format and then
527
yields the output of record_stream(), the main entry point to
530
Broadly this class has to unwrap two layers of iterators:
534
This is complicated by wishing to return type, iterator_for_type, but
535
getting the data for iterator_for_type when we find out type: we can't
536
simply pass a generator down to the NetworkRecordStream parser, instead
537
we have a little local state to seed each NetworkRecordStream instance,
538
and gather the type that we'll be yielding.
540
:ivar byte_stream: The byte stream being decoded.
541
:ivar stream_decoder: A pack parser used to decode the bytestream
542
:ivar current_type: The current type, used to join adjacent records of the
543
same type into a single stream.
544
:ivar first_bytes: The first bytes to give the next NetworkRecordStream.
402
def _byte_stream_to_stream(byte_stream):
403
"""Convert a byte stream into a format and a StreamSource stream.
405
:param byte_stream: A bytes iterator, as output by _stream_to_byte_stream.
406
:return: (RepositoryFormat, stream_generator)
547
def __init__(self, byte_stream):
548
"""Create a _ByteStreamDecoder."""
549
self.stream_decoder = pack.ContainerPushParser()
550
self.current_type = None
551
self.first_bytes = None
552
self.byte_stream = byte_stream
554
def iter_stream_decoder(self):
555
"""Iterate the contents of the pack from stream_decoder."""
556
# dequeue pending items
557
for record in self.stream_decoder.read_pending_records():
559
# Pull bytes of the wire, decode them to records, yield those records.
560
for bytes in self.byte_stream:
561
self.stream_decoder.accept_bytes(bytes)
562
for record in self.stream_decoder.read_pending_records():
565
def iter_substream_bytes(self):
566
if self.first_bytes is not None:
567
yield self.first_bytes
568
# If we run out of pack records, single the outer layer to stop.
569
self.first_bytes = None
570
for record in self.iter_pack_records:
408
stream_decoder = pack.ContainerPushParser()
410
"""Closure to return the substreams."""
411
# May have fully parsed records already.
412
for record in stream_decoder.read_pending_records():
571
413
record_names, record_bytes = record
572
414
record_name, = record_names
573
415
substream_type = record_name[0]
574
if substream_type != self.current_type:
575
# end of a substream, seed the next substream.
576
self.current_type = substream_type
577
self.first_bytes = record_bytes
581
def record_stream(self):
582
"""Yield substream_type, substream from the byte stream."""
584
# Make and consume sub generators, one per substream type:
585
while self.first_bytes is not None:
586
substream = NetworkRecordStream(self.iter_substream_bytes())
587
# after substream is fully consumed, self.current_type is set to
588
# the next type, and self.first_bytes is set to the matching bytes.
589
yield self.current_type, substream.read()
591
def seed_state(self):
592
"""Prepare the _ByteStreamDecoder to decode from the pack stream."""
593
# Set a single generator we can use to get data from the pack stream.
594
self.iter_pack_records = self.iter_stream_decoder()
595
# Seed the very first subiterator with content; after this each one
597
list(self.iter_substream_bytes())
600
def _byte_stream_to_stream(byte_stream):
601
"""Convert a byte stream into a format and a stream.
603
:param byte_stream: A bytes iterator, as output by _stream_to_byte_stream.
604
:return: (RepositoryFormat, stream_generator)
606
decoder = _ByteStreamDecoder(byte_stream)
416
substream = NetworkRecordStream([record_bytes])
417
yield substream_type, substream.read()
418
for bytes in byte_stream:
419
stream_decoder.accept_bytes(bytes)
420
for record in stream_decoder.read_pending_records():
421
record_names, record_bytes = record
422
record_name, = record_names
423
substream_type = record_name[0]
424
substream = NetworkRecordStream([record_bytes])
425
yield substream_type, substream.read()
607
426
for bytes in byte_stream:
608
decoder.stream_decoder.accept_bytes(bytes)
609
for record in decoder.stream_decoder.read_pending_records(max=1):
427
stream_decoder.accept_bytes(bytes)
428
for record in stream_decoder.read_pending_records(max=1):
610
429
record_names, src_format_name = record
611
430
src_format = network_format_registry.get(src_format_name)
612
return src_format, decoder.record_stream()
431
return src_format, record_stream()
615
434
class SmartServerRepositoryUnlock(SmartServerRepositoryRequest):
697
class SmartServerRepositoryInsertStreamLocked(SmartServerRepositoryRequest):
515
class SmartServerRepositoryInsertStream(SmartServerRepositoryRequest):
698
516
"""Insert a record stream from a RemoteSink into a repository.
700
518
This gets bytes pushed to it by the network infrastructure and turns that
701
519
into a bytes iterator using a thread. That is then processed by
702
520
_byte_stream_to_stream.
707
def do_repository_request(self, repository, resume_tokens, lock_token):
523
def do_repository_request(self, repository, resume_tokens):
708
524
"""StreamSink.insert_stream for a remote repository."""
709
repository.lock_write(token=lock_token)
710
self.do_insert_stream_request(repository, resume_tokens)
712
def do_insert_stream_request(self, repository, resume_tokens):
525
repository.lock_write()
713
526
tokens = [token for token in resume_tokens.split(' ') if token]
714
527
self.tokens = tokens
715
528
self.repository = repository
758
571
self.repository.unlock()
759
572
return SuccessfulSmartServerResponse(('ok', ))
762
class SmartServerRepositoryInsertStream_1_19(SmartServerRepositoryInsertStreamLocked):
763
"""Insert a record stream from a RemoteSink into a repository.
765
Same as SmartServerRepositoryInsertStreamLocked, except:
766
- the lock token argument is optional
767
- servers that implement this verb accept 'inventory-delta' records in the
773
def do_repository_request(self, repository, resume_tokens, lock_token=None):
774
"""StreamSink.insert_stream for a remote repository."""
775
SmartServerRepositoryInsertStreamLocked.do_repository_request(
776
self, repository, resume_tokens, lock_token)
779
class SmartServerRepositoryInsertStream(SmartServerRepositoryInsertStreamLocked):
780
"""Insert a record stream from a RemoteSink into an unlocked repository.
782
This is the same as SmartServerRepositoryInsertStreamLocked, except it
783
takes no lock_tokens; i.e. it works with an unlocked (or lock-free, e.g.
784
like pack format) repository.
789
def do_repository_request(self, repository, resume_tokens):
790
"""StreamSink.insert_stream for a remote repository."""
791
repository.lock_write()
792
self.do_insert_stream_request(repository, resume_tokens)
added
removed
bzrlib/smart/repository.py
