/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/plugins/upload/cmds.py

  • Committer: Jelmer Vernooij
  • Date: 2018-07-08 14:45:27 UTC
  • mto: This revision was merged to the branch mainline in revision 7036.
  • Revision ID: jelmer@jelmer.uk-20180708144527-codhlvdcdg9y0nji
Fix a bunch of merge tests.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/plugins/upload/cmds.py
 
1
# Copyright (C) 2011, 2012 Canonical Ltd
 
2
#
 
3
# This program is free software; you can redistribute it and/or modify
 
4
# it under the terms of the GNU General Public License as published by
 
5
# the Free Software Foundation; either version 2 of the License, or
 
6
# (at your option) any later version.
 
7
#
 
8
# This program is distributed in the hope that it will be useful,
 
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
11
# GNU General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
16
 
 
17
"""bzr-upload command implementations."""
 
18
 
 
19
from __future__ import absolute_import
 
20
 
 
21
from ... import (
 
22
    branch,
 
23
    commands,
 
24
    config,
 
25
    lazy_import,
 
26
    option,
 
27
    )
 
28
lazy_import.lazy_import(globals(), """
 
29
import stat
 
30
import sys
 
31
 
 
32
from breezy import (
 
33
    controldir,
 
34
    errors,
 
35
    globbing,
 
36
    ignores,
 
37
    revision,
 
38
    transport,
 
39
    urlutils,
 
40
    )
 
41
""")
 
42
 
 
43
from ...sixish import (
 
44
    text_type,
 
45
    )
 
46
 
 
47
auto_option = config.Option(
 
48
    'upload_auto', default=False, from_unicode=config.bool_from_store,
 
49
    help="""\
 
50
Whether upload should occur when the tip of the branch changes.
 
51
""")
 
52
auto_quiet_option = config.Option(
 
53
    'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
 
54
    help="""\
 
55
Whether upload should occur quietly.
 
56
""")
 
57
location_option = config.Option(
 
58
    'upload_location', default=None,
 
59
    help="""\
 
60
The url to upload the working tree to.
 
61
""")
 
62
revid_location_option = config.Option(
 
63
    'upload_revid_location', default=u'.bzr-upload.revid',
 
64
    help="""\
 
65
The relative path to be used to store the uploaded revid.
 
66
 
 
67
The only bzr-related info uploaded with the working tree is the corresponding
 
68
revision id. The uploaded working tree is not linked to any other bzr data.
 
69
 
 
70
If the layout of your remote server is such that you can't write in the
 
71
root directory but only in the directories inside that root, you will need
 
72
to use the 'upload_revid_location' configuration variable to specify the
 
73
relative path to be used. That configuration variable can be specified in
 
74
locations.conf or branch.conf.
 
75
 
 
76
For example, given the following layout:
 
77
 
 
78
  Project/
 
79
    private/
 
80
    public/
 
81
 
 
82
you may have write access in 'private' and 'public' but in 'Project'
 
83
itself. In that case, you can add the following in your locations.conf or
 
84
branch.conf file:
 
85
 
 
86
  upload_revid_location = private/.bzr-upload.revid
 
87
""")
 
88
 
 
89
 
 
90
# FIXME: Add more tests around invalid paths or relative paths that doesn't
 
91
# exist on remote (if only to get proper error messages) for
 
92
# 'upload_revid_location'
 
93
 
 
94
 
 
95
class BzrUploader(object):
 
96
 
 
97
    def __init__(self, branch, to_transport, outf, tree, rev_id,
 
98
                 quiet=False):
 
99
        self.branch = branch
 
100
        self.to_transport = to_transport
 
101
        self.outf = outf
 
102
        self.tree = tree
 
103
        self.rev_id = rev_id
 
104
        self.quiet = quiet
 
105
        self._pending_deletions = []
 
106
        self._pending_renames = []
 
107
        self._uploaded_revid = None
 
108
        self._ignored = None
 
109
 
 
110
    def _up_stat(self, relpath):
 
111
        return self.to_transport.stat(urlutils.escape(relpath))
 
112
 
 
113
    def _up_rename(self, old_path, new_path):
 
114
        return self.to_transport.rename(urlutils.escape(old_path),
 
115
                                        urlutils.escape(new_path))
 
116
 
 
117
    def _up_delete(self, relpath):
 
118
        return self.to_transport.delete(urlutils.escape(relpath))
 
119
 
 
120
    def _up_delete_tree(self, relpath):
 
121
        return self.to_transport.delete_tree(urlutils.escape(relpath))
 
122
 
 
123
    def _up_mkdir(self, relpath, mode):
 
124
        return self.to_transport.mkdir(urlutils.escape(relpath), mode)
 
125
 
 
126
    def _up_rmdir(self, relpath):
 
127
        return self.to_transport.rmdir(urlutils.escape(relpath))
 
128
 
 
129
    def _up_put_bytes(self, relpath, bytes, mode):
 
130
        self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
 
131
 
 
132
    def _up_get_bytes(self, relpath):
 
133
        return self.to_transport.get_bytes(urlutils.escape(relpath))
 
134
 
 
135
    def set_uploaded_revid(self, rev_id):
 
136
        # XXX: Add tests for concurrent updates, etc.
 
137
        revid_path = self.branch.get_config_stack().get('upload_revid_location')
 
138
        self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
 
139
        self._uploaded_revid = rev_id
 
140
 
 
141
    def get_uploaded_revid(self):
 
142
        if self._uploaded_revid is None:
 
143
            revid_path = self.branch.get_config_stack(
 
144
                ).get('upload_revid_location')
 
145
            try:
 
146
                self._uploaded_revid = self._up_get_bytes(revid_path)
 
147
            except errors.NoSuchFile:
 
148
                # We have not uploaded to here.
 
149
                self._uploaded_revid = revision.NULL_REVISION
 
150
        return self._uploaded_revid
 
151
 
 
152
    def _get_ignored(self):
 
153
        if self._ignored is None:
 
154
            try:
 
155
                ignore_file_path = '.bzrignore-upload'
 
156
                ignore_file = self.tree.get_file(ignore_file_path)
 
157
            except errors.NoSuchFile:
 
158
                ignored_patterns = []
 
159
            else:
 
160
                ignored_patterns = ignores.parse_ignore_file(ignore_file)
 
161
            self._ignored = globbing.Globster(ignored_patterns)
 
162
        return self._ignored
 
163
 
 
164
    def is_ignored(self, relpath):
 
165
        glob = self._get_ignored()
 
166
        ignored = glob.match(relpath)
 
167
        import os
 
168
        if not ignored:
 
169
            # We still need to check that all parents are not ignored
 
170
            dir = os.path.dirname(relpath)
 
171
            while dir and not ignored:
 
172
                ignored = glob.match(dir)
 
173
                if not ignored:
 
174
                    dir = os.path.dirname(dir)
 
175
        return ignored
 
176
 
 
177
    def upload_file(self, old_relpath, new_relpath, id, mode=None):
 
178
        if mode is None:
 
179
            if self.tree.is_executable(new_relpath, id):
 
180
                mode = 0o775
 
181
            else:
 
182
                mode = 0o664
 
183
        if not self.quiet:
 
184
            self.outf.write('Uploading %s\n' % old_relpath)
 
185
        self._up_put_bytes(old_relpath, self.tree.get_file_text(new_relpath, id), mode)
 
186
 
 
187
    def upload_file_robustly(self, relpath, id, mode=None):
 
188
        """Upload a file, clearing the way on the remote side.
 
189
 
 
190
        When doing a full upload, it may happen that a directory exists where
 
191
        we want to put our file.
 
192
        """
 
193
        try:
 
194
            st = self._up_stat(relpath)
 
195
            if stat.S_ISDIR(st.st_mode):
 
196
                # A simple rmdir may not be enough
 
197
                if not self.quiet:
 
198
                    self.outf.write('Clearing %s/%s\n' % (
 
199
                            self.to_transport.external_url(), relpath))
 
200
                self._up_delete_tree(relpath)
 
201
        except errors.PathError:
 
202
            pass
 
203
        self.upload_file(relpath, relpath, id, mode)
 
204
 
 
205
    def make_remote_dir(self, relpath, mode=None):
 
206
        if mode is None:
 
207
            mode = 0o775
 
208
        self._up_mkdir(relpath, mode)
 
209
 
 
210
    def make_remote_dir_robustly(self, relpath, mode=None):
 
211
        """Create a remote directory, clearing the way on the remote side.
 
212
 
 
213
        When doing a full upload, it may happen that a file exists where we
 
214
        want to create our directory.
 
215
        """
 
216
        try:
 
217
            st = self._up_stat(relpath)
 
218
            if not stat.S_ISDIR(st.st_mode):
 
219
                if not self.quiet:
 
220
                    self.outf.write('Deleting %s/%s\n' % (
 
221
                            self.to_transport.external_url(), relpath))
 
222
                self._up_delete(relpath)
 
223
            else:
 
224
                # Ok the remote dir already exists, nothing to do
 
225
                return
 
226
        except errors.PathError:
 
227
            pass
 
228
        self.make_remote_dir(relpath, mode)
 
229
 
 
230
    def delete_remote_file(self, relpath):
 
231
        if not self.quiet:
 
232
            self.outf.write('Deleting %s\n' % relpath)
 
233
        self._up_delete(relpath)
 
234
 
 
235
    def delete_remote_dir(self, relpath):
 
236
        if not self.quiet:
 
237
            self.outf.write('Deleting %s\n' % relpath)
 
238
        self._up_rmdir(relpath)
 
239
        # XXX: Add a test where a subdir is ignored but we still want to
 
240
        # delete the dir -- vila 100106
 
241
 
 
242
    def delete_remote_dir_maybe(self, relpath):
 
243
        """Try to delete relpath, keeping failures to retry later."""
 
244
        try:
 
245
            self._up_rmdir(relpath)
 
246
        # any kind of PathError would be OK, though we normally expect
 
247
        # DirectoryNotEmpty
 
248
        except errors.PathError:
 
249
            self._pending_deletions.append(relpath)
 
250
 
 
251
    def finish_deletions(self):
 
252
        if self._pending_deletions:
 
253
            # Process the previously failed deletions in reverse order to
 
254
            # delete children before parents
 
255
            for relpath in reversed(self._pending_deletions):
 
256
                self._up_rmdir(relpath)
 
257
            # The following shouldn't be needed since we use it once per
 
258
            # upload, but better safe than sorry ;-)
 
259
            self._pending_deletions = []
 
260
 
 
261
    def rename_remote(self, old_relpath, new_relpath):
 
262
        """Rename a remote file or directory taking care of collisions.
 
263
 
 
264
        To avoid collisions during bulk renames, each renamed target is
 
265
        temporarily assigned a unique name. When all renames have been done,
 
266
        each target get its proper name.
 
267
        """
 
268
        # We generate a sufficiently random name to *assume* that
 
269
        # no collisions will occur and don't worry about it (nor
 
270
        # handle it).
 
271
        import os
 
272
        import random
 
273
        import time
 
274
 
 
275
        stamp = '.tmp.%.9f.%d.%d' % (time.time(),
 
276
                                     os.getpid(),
 
277
                                     random.randint(0, 0x7FFFFFFF))
 
278
        if not self.quiet:
 
279
            self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
 
280
        self._up_rename(old_relpath, stamp)
 
281
        self._pending_renames.append((stamp, new_relpath))
 
282
 
 
283
    def finish_renames(self):
 
284
        for (stamp, new_path) in self._pending_renames:
 
285
            self._up_rename(stamp, new_path)
 
286
        # The following shouldn't be needed since we use it once per upload,
 
287
        # but better safe than sorry ;-)
 
288
        self._pending_renames = []
 
289
 
 
290
    def upload_full_tree(self):
 
291
        self.to_transport.ensure_base() # XXX: Handle errors (add
 
292
                                        # --create-prefix option ?)
 
293
        with self.tree.lock_read():
 
294
            for relpath, ie in self.tree.iter_entries_by_dir():
 
295
                if relpath in ('', '.bzrignore', '.bzrignore-upload'):
 
296
                    # skip root ('')
 
297
                    # .bzrignore and .bzrignore-upload have no meaning outside
 
298
                    # a working tree so do not upload them
 
299
                    continue
 
300
                if self.is_ignored(relpath):
 
301
                    if not self.quiet:
 
302
                        self.outf.write('Ignoring %s\n' % relpath)
 
303
                    continue
 
304
                if ie.kind == 'file':
 
305
                    self.upload_file_robustly(relpath, ie.file_id)
 
306
                elif ie.kind == 'directory':
 
307
                    self.make_remote_dir_robustly(relpath)
 
308
                elif ie.kind == 'symlink':
 
309
                    if not self.quiet:
 
310
                        target = self.tree.path_content_summary(relpath)[3]
 
311
                        self.outf.write('Not uploading symlink %s -> %s\n'
 
312
                                        % (relpath, target))
 
313
                else:
 
314
                    raise NotImplementedError
 
315
            self.set_uploaded_revid(self.rev_id)
 
316
 
 
317
    def upload_tree(self):
 
318
        # If we can't find the revid file on the remote location, upload the
 
319
        # full tree instead
 
320
        rev_id = self.get_uploaded_revid()
 
321
 
 
322
        if rev_id == revision.NULL_REVISION:
 
323
            if not self.quiet:
 
324
                self.outf.write('No uploaded revision id found,'
 
325
                                ' switching to full upload\n')
 
326
            self.upload_full_tree()
 
327
            # We're done
 
328
            return
 
329
 
 
330
        # Check if the revision hasn't already been uploaded
 
331
        if rev_id == self.rev_id:
 
332
            if not self.quiet:
 
333
                self.outf.write('Remote location already up to date\n')
 
334
 
 
335
        from_tree = self.branch.repository.revision_tree(rev_id)
 
336
        self.to_transport.ensure_base() # XXX: Handle errors (add
 
337
                                        # --create-prefix option ?)
 
338
        changes = self.tree.changes_from(from_tree)
 
339
        with self.tree.lock_read():
 
340
            for (path, id, kind) in changes.removed:
 
341
                if self.is_ignored(path):
 
342
                    if not self.quiet:
 
343
                        self.outf.write('Ignoring %s\n' % path)
 
344
                    continue
 
345
                if kind is 'file':
 
346
                    self.delete_remote_file(path)
 
347
                elif kind is  'directory':
 
348
                    self.delete_remote_dir_maybe(path)
 
349
                elif kind == 'symlink':
 
350
                    if not self.quiet:
 
351
                        target = self.tree.path_content_summary(path)[3]
 
352
                        self.outf.write('Not deleting remote symlink %s -> %s\n'
 
353
                                        % (path, target))
 
354
                else:
 
355
                    raise NotImplementedError
 
356
 
 
357
            for (old_path, new_path, id, kind,
 
358
                 content_change, exec_change) in changes.renamed:
 
359
                if self.is_ignored(old_path) and self.is_ignored(new_path):
 
360
                    if not self.quiet:
 
361
                        self.outf.write('Ignoring %s\n' % old_path)
 
362
                        self.outf.write('Ignoring %s\n' % new_path)
 
363
                    continue
 
364
                if content_change:
 
365
                    # We update the old_path content because renames and
 
366
                    # deletions are differed.
 
367
                    self.upload_file(old_path, new_path, id)
 
368
                if kind == 'symlink':
 
369
                    if not self.quiet:
 
370
                        self.outf.write('Not renaming remote symlink %s to %s\n'
 
371
                                        % (old_path, new_path))
 
372
                else:
 
373
                    self.rename_remote(old_path, new_path)
 
374
            self.finish_renames()
 
375
            self.finish_deletions()
 
376
 
 
377
            for (path, id, old_kind, new_kind) in changes.kind_changed:
 
378
                if self.is_ignored(path):
 
379
                    if not self.quiet:
 
380
                        self.outf.write('Ignoring %s\n' % path)
 
381
                    continue
 
382
                if old_kind == 'file':
 
383
                    self.delete_remote_file(path)
 
384
                elif old_kind ==  'directory':
 
385
                    self.delete_remote_dir(path)
 
386
                else:
 
387
                    raise NotImplementedError
 
388
 
 
389
                if new_kind == 'file':
 
390
                    self.upload_file(path, path, id)
 
391
                elif new_kind is 'directory':
 
392
                    self.make_remote_dir(path)
 
393
                else:
 
394
                    raise NotImplementedError
 
395
 
 
396
            for (path, id, kind) in changes.added:
 
397
                if self.is_ignored(path):
 
398
                    if not self.quiet:
 
399
                        self.outf.write('Ignoring %s\n' % path)
 
400
                    continue
 
401
                if kind == 'file':
 
402
                    self.upload_file(path, path, id)
 
403
                elif kind == 'directory':
 
404
                    self.make_remote_dir(path)
 
405
                elif kind == 'symlink':
 
406
                    if not self.quiet:
 
407
                        target = self.tree.path_content_summary(path)[3]
 
408
                        self.outf.write('Not uploading symlink %s -> %s\n'
 
409
                                        % (path, target))
 
410
                else:
 
411
                    raise NotImplementedError
 
412
 
 
413
            # XXX: Add a test for exec_change
 
414
            for (path, id, kind,
 
415
                 content_change, exec_change) in changes.modified:
 
416
                if self.is_ignored(path):
 
417
                    if not self.quiet:
 
418
                        self.outf.write('Ignoring %s\n' % path)
 
419
                    continue
 
420
                if kind == 'file':
 
421
                    self.upload_file(path, path, id)
 
422
                else:
 
423
                    raise NotImplementedError
 
424
 
 
425
            self.set_uploaded_revid(self.rev_id)
 
426
 
 
427
 
 
428
class CannotUploadToWorkingTree(errors.BzrCommandError):
 
429
 
 
430
    _fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
 
431
 
 
432
 
 
433
class DivergedUploadedTree(errors.BzrCommandError):
 
434
 
 
435
    _fmt = ("Your branch (%(revid)s)"
 
436
            " and the uploaded tree (%(uploaded_revid)s) have diverged: ")
 
437
 
 
438
 
 
439
class cmd_upload(commands.Command):
 
440
    """Upload a working tree, as a whole or incrementally.
 
441
 
 
442
    If no destination is specified use the last one used.
 
443
    If no revision is specified upload the changes since the last upload.
 
444
 
 
445
    Changes include files added, renamed, modified or removed.
 
446
    """
 
447
    _see_also = ['plugins/upload']
 
448
    takes_args = ['location?']
 
449
    takes_options = [
 
450
        'revision',
 
451
        'remember',
 
452
        'overwrite',
 
453
        option.Option('full', 'Upload the full working tree.'),
 
454
        option.Option('quiet', 'Do not output what is being done.',
 
455
                       short_name='q'),
 
456
        option.Option('directory',
 
457
                      help='Branch to upload from, '
 
458
                      'rather than the one containing the working directory.',
 
459
                      short_name='d',
 
460
                      type=text_type,
 
461
                      ),
 
462
        option.Option('auto',
 
463
                      'Trigger an upload from this branch whenever the tip '
 
464
                      'revision changes.')
 
465
       ]
 
466
 
 
467
    def run(self, location=None, full=False, revision=None, remember=None,
 
468
            directory=None, quiet=False, auto=None, overwrite=False
 
469
            ):
 
470
        if directory is None:
 
471
            directory = u'.'
 
472
 
 
473
        (wt, branch,
 
474
         relpath) = controldir.ControlDir.open_containing_tree_or_branch(
 
475
             directory)
 
476
 
 
477
        if wt:
 
478
            wt.lock_read()
 
479
            locked = wt
 
480
        else:
 
481
            branch.lock_read()
 
482
            locked = branch
 
483
        try:
 
484
            if wt:
 
485
                changes = wt.changes_from(wt.basis_tree())
 
486
 
 
487
                if revision is None and  changes.has_changed():
 
488
                    raise errors.UncommittedChanges(wt)
 
489
 
 
490
            conf = branch.get_config_stack()
 
491
            if location is None:
 
492
                stored_loc = conf.get('upload_location')
 
493
                if stored_loc is None:
 
494
                    raise errors.BzrCommandError(
 
495
                        'No upload location known or specified.')
 
496
                else:
 
497
                    # FIXME: Not currently tested
 
498
                    display_url = urlutils.unescape_for_display(stored_loc,
 
499
                            self.outf.encoding)
 
500
                    self.outf.write("Using saved location: %s\n" % display_url)
 
501
                    location = stored_loc
 
502
 
 
503
            to_transport = transport.get_transport(location)
 
504
 
 
505
            # Check that we are not uploading to a existing working tree.
 
506
            try:
 
507
                to_bzr_dir = controldir.ControlDir.open_from_transport(
 
508
                        to_transport)
 
509
                has_wt = to_bzr_dir.has_workingtree()
 
510
            except errors.NotBranchError:
 
511
                has_wt = False
 
512
            except errors.NotLocalUrl:
 
513
                # The exception raised is a bit weird... but that's life.
 
514
                has_wt = True
 
515
 
 
516
            if has_wt:
 
517
                raise CannotUploadToWorkingTree(url=location)
 
518
            if revision is None:
 
519
                rev_id = branch.last_revision()
 
520
            else:
 
521
                if len(revision) != 1:
 
522
                    raise errors.BzrCommandError(
 
523
                        'bzr upload --revision takes exactly 1 argument')
 
524
                rev_id = revision[0].in_history(branch).rev_id
 
525
 
 
526
            tree = branch.repository.revision_tree(rev_id)
 
527
 
 
528
            uploader = BzrUploader(branch, to_transport, self.outf, tree,
 
529
                                   rev_id, quiet=quiet)
 
530
 
 
531
            if not overwrite:
 
532
                prev_uploaded_rev_id = uploader.get_uploaded_revid()
 
533
                graph = branch.repository.get_graph()
 
534
                if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
 
535
                    raise DivergedUploadedTree(
 
536
                        revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
 
537
            if full:
 
538
                uploader.upload_full_tree()
 
539
            else:
 
540
                uploader.upload_tree()
 
541
        finally:
 
542
            locked.unlock()
 
543
 
 
544
        # We uploaded successfully, remember it
 
545
        branch.lock_write()
 
546
        try:
 
547
            upload_location = conf.get('upload_location')
 
548
            if upload_location is None or remember:
 
549
                conf.set('upload_location',
 
550
                         urlutils.unescape(to_transport.base))
 
551
            if auto is not None:
 
552
                conf.set('upload_auto', auto)
 
553
        finally:
 
554
            branch.unlock()
 
555