/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/plugins/upload/cmds.py

  • Committer: Breezy landing bot
  • Author(s): Colin Watson
  • Date: 2020-11-16 21:47:08 UTC
  • mfrom: (7521.1.1 remove-lp-workaround)
  • Revision ID: breezy.the.bot@gmail.com-20201116214708-jos209mgxi41oy15
Remove breezy.git workaround for bazaar.launchpad.net.

Merged from https://code.launchpad.net/~cjwatson/brz/remove-lp-workaround/+merge/393710

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/plugins/upload/cmds.py
 
1
# Copyright (C) 2011, 2012 Canonical Ltd
 
2
#
 
3
# This program is free software; you can redistribute it and/or modify
 
4
# it under the terms of the GNU General Public License as published by
 
5
# the Free Software Foundation; either version 2 of the License, or
 
6
# (at your option) any later version.
 
7
#
 
8
# This program is distributed in the hope that it will be useful,
 
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
11
# GNU General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
16
 
 
17
"""bzr-upload command implementations."""
 
18
 
 
19
from ... import (
 
20
    commands,
 
21
    config,
 
22
    errors,
 
23
    lazy_import,
 
24
    option,
 
25
    osutils,
 
26
    )
 
27
lazy_import.lazy_import(globals(), """
 
28
import stat
 
29
 
 
30
from breezy import (
 
31
    controldir,
 
32
    globbing,
 
33
    ignores,
 
34
    revision,
 
35
    transport,
 
36
    urlutils,
 
37
    )
 
38
""")
 
39
 
 
40
 
 
41
auto_option = config.Option(
 
42
    'upload_auto', default=False, from_unicode=config.bool_from_store,
 
43
    help="""\
 
44
Whether upload should occur when the tip of the branch changes.
 
45
""")
 
46
auto_quiet_option = config.Option(
 
47
    'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
 
48
    help="""\
 
49
Whether upload should occur quietly.
 
50
""")
 
51
location_option = config.Option(
 
52
    'upload_location', default=None,
 
53
    help="""\
 
54
The url to upload the working tree to.
 
55
""")
 
56
revid_location_option = config.Option(
 
57
    'upload_revid_location', default=u'.bzr-upload.revid',
 
58
    help="""\
 
59
The relative path to be used to store the uploaded revid.
 
60
 
 
61
The only bzr-related info uploaded with the working tree is the corresponding
 
62
revision id. The uploaded working tree is not linked to any other bzr data.
 
63
 
 
64
If the layout of your remote server is such that you can't write in the
 
65
root directory but only in the directories inside that root, you will need
 
66
to use the 'upload_revid_location' configuration variable to specify the
 
67
relative path to be used. That configuration variable can be specified in
 
68
locations.conf or branch.conf.
 
69
 
 
70
For example, given the following layout:
 
71
 
 
72
  Project/
 
73
    private/
 
74
    public/
 
75
 
 
76
you may have write access in 'private' and 'public' but in 'Project'
 
77
itself. In that case, you can add the following in your locations.conf or
 
78
branch.conf file:
 
79
 
 
80
  upload_revid_location = private/.bzr-upload.revid
 
81
""")
 
82
 
 
83
 
 
84
# FIXME: Add more tests around invalid paths or relative paths that doesn't
 
85
# exist on remote (if only to get proper error messages) for
 
86
# 'upload_revid_location'
 
87
 
 
88
 
 
89
class BzrUploader(object):
 
90
 
 
91
    def __init__(self, branch, to_transport, outf, tree, rev_id,
 
92
                 quiet=False):
 
93
        self.branch = branch
 
94
        self.to_transport = to_transport
 
95
        self.outf = outf
 
96
        self.tree = tree
 
97
        self.rev_id = rev_id
 
98
        self.quiet = quiet
 
99
        self._pending_deletions = []
 
100
        self._pending_renames = []
 
101
        self._uploaded_revid = None
 
102
        self._ignored = None
 
103
 
 
104
    def _up_stat(self, relpath):
 
105
        return self.to_transport.stat(urlutils.escape(relpath))
 
106
 
 
107
    def _up_rename(self, old_path, new_path):
 
108
        return self.to_transport.rename(urlutils.escape(old_path),
 
109
                                        urlutils.escape(new_path))
 
110
 
 
111
    def _up_delete(self, relpath):
 
112
        return self.to_transport.delete(urlutils.escape(relpath))
 
113
 
 
114
    def _up_delete_tree(self, relpath):
 
115
        return self.to_transport.delete_tree(urlutils.escape(relpath))
 
116
 
 
117
    def _up_mkdir(self, relpath, mode):
 
118
        return self.to_transport.mkdir(urlutils.escape(relpath), mode)
 
119
 
 
120
    def _up_rmdir(self, relpath):
 
121
        return self.to_transport.rmdir(urlutils.escape(relpath))
 
122
 
 
123
    def _up_put_bytes(self, relpath, bytes, mode):
 
124
        self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
 
125
 
 
126
    def _up_get_bytes(self, relpath):
 
127
        return self.to_transport.get_bytes(urlutils.escape(relpath))
 
128
 
 
129
    def set_uploaded_revid(self, rev_id):
 
130
        # XXX: Add tests for concurrent updates, etc.
 
131
        revid_path = self.branch.get_config_stack().get('upload_revid_location')
 
132
        self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
 
133
        self._uploaded_revid = rev_id
 
134
 
 
135
    def get_uploaded_revid(self):
 
136
        if self._uploaded_revid is None:
 
137
            revid_path = self.branch.get_config_stack(
 
138
                ).get('upload_revid_location')
 
139
            try:
 
140
                self._uploaded_revid = self._up_get_bytes(revid_path)
 
141
            except errors.NoSuchFile:
 
142
                # We have not uploaded to here.
 
143
                self._uploaded_revid = revision.NULL_REVISION
 
144
        return self._uploaded_revid
 
145
 
 
146
    def _get_ignored(self):
 
147
        if self._ignored is None:
 
148
            try:
 
149
                ignore_file_path = '.bzrignore-upload'
 
150
                ignore_file = self.tree.get_file(ignore_file_path)
 
151
            except errors.NoSuchFile:
 
152
                ignored_patterns = []
 
153
            else:
 
154
                ignored_patterns = ignores.parse_ignore_file(ignore_file)
 
155
            self._ignored = globbing.Globster(ignored_patterns)
 
156
        return self._ignored
 
157
 
 
158
    def is_ignored(self, relpath):
 
159
        glob = self._get_ignored()
 
160
        ignored = glob.match(relpath)
 
161
        import os
 
162
        if not ignored:
 
163
            # We still need to check that all parents are not ignored
 
164
            dir = os.path.dirname(relpath)
 
165
            while dir and not ignored:
 
166
                ignored = glob.match(dir)
 
167
                if not ignored:
 
168
                    dir = os.path.dirname(dir)
 
169
        return ignored
 
170
 
 
171
    def upload_file(self, old_relpath, new_relpath, mode=None):
 
172
        if mode is None:
 
173
            if self.tree.is_executable(new_relpath):
 
174
                mode = 0o775
 
175
            else:
 
176
                mode = 0o664
 
177
        if not self.quiet:
 
178
            self.outf.write('Uploading %s\n' % old_relpath)
 
179
        self._up_put_bytes(
 
180
            old_relpath, self.tree.get_file_text(new_relpath), mode)
 
181
 
 
182
    def _force_clear(self, relpath):
 
183
        try:
 
184
            st = self._up_stat(relpath)
 
185
            if stat.S_ISDIR(st.st_mode):
 
186
                # A simple rmdir may not be enough
 
187
                if not self.quiet:
 
188
                    self.outf.write('Clearing %s/%s\n' % (
 
189
                        self.to_transport.external_url(), relpath))
 
190
                self._up_delete_tree(relpath)
 
191
            elif stat.S_ISLNK(st.st_mode):
 
192
                if not self.quiet:
 
193
                    self.outf.write('Clearing %s/%s\n' % (
 
194
                        self.to_transport.external_url(), relpath))
 
195
                self._up_delete(relpath)
 
196
        except errors.PathError:
 
197
            pass
 
198
 
 
199
    def upload_file_robustly(self, relpath, mode=None):
 
200
        """Upload a file, clearing the way on the remote side.
 
201
 
 
202
        When doing a full upload, it may happen that a directory exists where
 
203
        we want to put our file.
 
204
        """
 
205
        self._force_clear(relpath)
 
206
        self.upload_file(relpath, relpath, mode)
 
207
 
 
208
    def upload_symlink(self, relpath, target):
 
209
        self.to_transport.symlink(target, relpath)
 
210
 
 
211
    def upload_symlink_robustly(self, relpath, target):
 
212
        """Handle uploading symlinks.
 
213
        """
 
214
        self._force_clear(relpath)
 
215
        # Target might not be there at this time; dummy file should be
 
216
        # overwritten at some point, possibly by another upload.
 
217
        target = osutils.normpath(osutils.pathjoin(
 
218
            osutils.dirname(relpath),
 
219
            target)
 
220
        )
 
221
        self.upload_symlink(relpath, target)
 
222
 
 
223
    def make_remote_dir(self, relpath, mode=None):
 
224
        if mode is None:
 
225
            mode = 0o775
 
226
        self._up_mkdir(relpath, mode)
 
227
 
 
228
    def make_remote_dir_robustly(self, relpath, mode=None):
 
229
        """Create a remote directory, clearing the way on the remote side.
 
230
 
 
231
        When doing a full upload, it may happen that a file exists where we
 
232
        want to create our directory.
 
233
        """
 
234
        try:
 
235
            st = self._up_stat(relpath)
 
236
            if not stat.S_ISDIR(st.st_mode):
 
237
                if not self.quiet:
 
238
                    self.outf.write('Deleting %s/%s\n' % (
 
239
                        self.to_transport.external_url(), relpath))
 
240
                self._up_delete(relpath)
 
241
            else:
 
242
                # Ok the remote dir already exists, nothing to do
 
243
                return
 
244
        except errors.PathError:
 
245
            pass
 
246
        self.make_remote_dir(relpath, mode)
 
247
 
 
248
    def delete_remote_file(self, relpath):
 
249
        if not self.quiet:
 
250
            self.outf.write('Deleting %s\n' % relpath)
 
251
        self._up_delete(relpath)
 
252
 
 
253
    def delete_remote_dir(self, relpath):
 
254
        if not self.quiet:
 
255
            self.outf.write('Deleting %s\n' % relpath)
 
256
        self._up_rmdir(relpath)
 
257
        # XXX: Add a test where a subdir is ignored but we still want to
 
258
        # delete the dir -- vila 100106
 
259
 
 
260
    def delete_remote_dir_maybe(self, relpath):
 
261
        """Try to delete relpath, keeping failures to retry later."""
 
262
        try:
 
263
            self._up_rmdir(relpath)
 
264
        # any kind of PathError would be OK, though we normally expect
 
265
        # DirectoryNotEmpty
 
266
        except errors.PathError:
 
267
            self._pending_deletions.append(relpath)
 
268
 
 
269
    def finish_deletions(self):
 
270
        if self._pending_deletions:
 
271
            # Process the previously failed deletions in reverse order to
 
272
            # delete children before parents
 
273
            for relpath in reversed(self._pending_deletions):
 
274
                self._up_rmdir(relpath)
 
275
            # The following shouldn't be needed since we use it once per
 
276
            # upload, but better safe than sorry ;-)
 
277
            self._pending_deletions = []
 
278
 
 
279
    def rename_remote(self, old_relpath, new_relpath):
 
280
        """Rename a remote file or directory taking care of collisions.
 
281
 
 
282
        To avoid collisions during bulk renames, each renamed target is
 
283
        temporarily assigned a unique name. When all renames have been done,
 
284
        each target get its proper name.
 
285
        """
 
286
        # We generate a sufficiently random name to *assume* that
 
287
        # no collisions will occur and don't worry about it (nor
 
288
        # handle it).
 
289
        import os
 
290
        import random
 
291
        import time
 
292
 
 
293
        stamp = '.tmp.%.9f.%d.%d' % (time.time(),
 
294
                                     os.getpid(),
 
295
                                     random.randint(0, 0x7FFFFFFF))
 
296
        if not self.quiet:
 
297
            self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
 
298
        self._up_rename(old_relpath, stamp)
 
299
        self._pending_renames.append((stamp, new_relpath))
 
300
 
 
301
    def finish_renames(self):
 
302
        for (stamp, new_path) in self._pending_renames:
 
303
            self._up_rename(stamp, new_path)
 
304
        # The following shouldn't be needed since we use it once per upload,
 
305
        # but better safe than sorry ;-)
 
306
        self._pending_renames = []
 
307
 
 
308
    def upload_full_tree(self):
 
309
        self.to_transport.ensure_base()  # XXX: Handle errors (add
 
310
        # --create-prefix option ?)
 
311
        with self.tree.lock_read():
 
312
            for relpath, ie in self.tree.iter_entries_by_dir():
 
313
                if relpath in ('', '.bzrignore', '.bzrignore-upload'):
 
314
                    # skip root ('')
 
315
                    # .bzrignore and .bzrignore-upload have no meaning outside
 
316
                    # a working tree so do not upload them
 
317
                    continue
 
318
                if self.is_ignored(relpath):
 
319
                    if not self.quiet:
 
320
                        self.outf.write('Ignoring %s\n' % relpath)
 
321
                    continue
 
322
                if ie.kind == 'file':
 
323
                    self.upload_file_robustly(relpath)
 
324
                elif ie.kind == 'symlink':
 
325
                    try:
 
326
                        self.upload_symlink_robustly(
 
327
                            relpath, ie.symlink_target)
 
328
                    except errors.TransportNotPossible:
 
329
                        if not self.quiet:
 
330
                            target = self.tree.path_content_summary(relpath)[3]
 
331
                            self.outf.write('Not uploading symlink %s -> %s\n'
 
332
                                            % (relpath, target))
 
333
                elif ie.kind == 'directory':
 
334
                    self.make_remote_dir_robustly(relpath)
 
335
                else:
 
336
                    raise NotImplementedError
 
337
            self.set_uploaded_revid(self.rev_id)
 
338
 
 
339
    def upload_tree(self):
 
340
        # If we can't find the revid file on the remote location, upload the
 
341
        # full tree instead
 
342
        rev_id = self.get_uploaded_revid()
 
343
 
 
344
        if rev_id == revision.NULL_REVISION:
 
345
            if not self.quiet:
 
346
                self.outf.write('No uploaded revision id found,'
 
347
                                ' switching to full upload\n')
 
348
            self.upload_full_tree()
 
349
            # We're done
 
350
            return
 
351
 
 
352
        # Check if the revision hasn't already been uploaded
 
353
        if rev_id == self.rev_id:
 
354
            if not self.quiet:
 
355
                self.outf.write('Remote location already up to date\n')
 
356
 
 
357
        from_tree = self.branch.repository.revision_tree(rev_id)
 
358
        self.to_transport.ensure_base()  # XXX: Handle errors (add
 
359
        # --create-prefix option ?)
 
360
        changes = self.tree.changes_from(from_tree)
 
361
        with self.tree.lock_read():
 
362
            for change in changes.removed:
 
363
                if self.is_ignored(change.path[0]):
 
364
                    if not self.quiet:
 
365
                        self.outf.write('Ignoring %s\n' % change.path[0])
 
366
                    continue
 
367
                if change.kind[0] == 'file':
 
368
                    self.delete_remote_file(change.path[0])
 
369
                elif change.kind[0] == 'directory':
 
370
                    self.delete_remote_dir_maybe(change.path[0])
 
371
                elif change.kind[0] == 'symlink':
 
372
                    self.delete_remote_file(change.path[0])
 
373
                else:
 
374
                    raise NotImplementedError
 
375
 
 
376
            for change in changes.renamed:
 
377
                if self.is_ignored(change.path[0]) and self.is_ignored(change.path[1]):
 
378
                    if not self.quiet:
 
379
                        self.outf.write('Ignoring %s\n' % change.path[0])
 
380
                        self.outf.write('Ignoring %s\n' % change.path[1])
 
381
                    continue
 
382
                if change.changed_content:
 
383
                    # We update the change.path[0] content because renames and
 
384
                    # deletions are differed.
 
385
                    self.upload_file(change.path[0], change.path[1])
 
386
                self.rename_remote(change.path[0], change.path[1])
 
387
            self.finish_renames()
 
388
            self.finish_deletions()
 
389
 
 
390
            for change in changes.kind_changed:
 
391
                if self.is_ignored(change.path[1]):
 
392
                    if not self.quiet:
 
393
                        self.outf.write('Ignoring %s\n' % change.path[1])
 
394
                    continue
 
395
                if change.kind[0] in ('file', 'symlink'):
 
396
                    self.delete_remote_file(change.path[0])
 
397
                elif change.kind[0] == 'directory':
 
398
                    self.delete_remote_dir(change.path[0])
 
399
                else:
 
400
                    raise NotImplementedError
 
401
 
 
402
                if change.kind[1] == 'file':
 
403
                    self.upload_file(change.path[1], change.path[1])
 
404
                elif change.kind[1] == 'symlink':
 
405
                    target = self.tree.get_symlink_target(change.path[1])
 
406
                    self.upload_symlink(change.path[1], target)
 
407
                elif change.kind[1] == 'directory':
 
408
                    self.make_remote_dir(change.path[1])
 
409
                else:
 
410
                    raise NotImplementedError
 
411
 
 
412
            for change in changes.added + changes.copied:
 
413
                if self.is_ignored(change.path[1]):
 
414
                    if not self.quiet:
 
415
                        self.outf.write('Ignoring %s\n' % change.path[1])
 
416
                    continue
 
417
                if change.kind[1] == 'file':
 
418
                    self.upload_file(change.path[1], change.path[1])
 
419
                elif change.kind[1] == 'directory':
 
420
                    self.make_remote_dir(change.path[1])
 
421
                elif change.kind[1] == 'symlink':
 
422
                    target = self.tree.get_symlink_target(change.path[1])
 
423
                    try:
 
424
                        self.upload_symlink(change.path[1], target)
 
425
                    except errors.TransportNotPossible:
 
426
                        if not self.quiet:
 
427
                            self.outf.write('Not uploading symlink %s -> %s\n'
 
428
                                            % (change.path[1], target))
 
429
                else:
 
430
                    raise NotImplementedError
 
431
 
 
432
            # XXX: Add a test for exec_change
 
433
            for change in changes.modified:
 
434
                if self.is_ignored(change.path[1]):
 
435
                    if not self.quiet:
 
436
                        self.outf.write('Ignoring %s\n' % change.path[1])
 
437
                    continue
 
438
                if change.kind[1] == 'file':
 
439
                    self.upload_file(change.path[1], change.path[1])
 
440
                elif change.kind[1] == 'symlink':
 
441
                    target = self.tree.get_symlink_target(change.path[1])
 
442
                    self.upload_symlink(change.path[1], target)
 
443
                else:
 
444
                    raise NotImplementedError
 
445
 
 
446
            self.set_uploaded_revid(self.rev_id)
 
447
 
 
448
 
 
449
class CannotUploadToWorkingTree(errors.CommandError):
 
450
 
 
451
    _fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
 
452
 
 
453
 
 
454
class DivergedUploadedTree(errors.CommandError):
 
455
 
 
456
    _fmt = ("Your branch (%(revid)s)"
 
457
            " and the uploaded tree (%(uploaded_revid)s) have diverged: ")
 
458
 
 
459
 
 
460
class cmd_upload(commands.Command):
 
461
    """Upload a working tree, as a whole or incrementally.
 
462
 
 
463
    If no destination is specified use the last one used.
 
464
    If no revision is specified upload the changes since the last upload.
 
465
 
 
466
    Changes include files added, renamed, modified or removed.
 
467
    """
 
468
    _see_also = ['plugins/upload']
 
469
    takes_args = ['location?']
 
470
    takes_options = [
 
471
        'revision',
 
472
        'remember',
 
473
        'overwrite',
 
474
        option.Option('full', 'Upload the full working tree.'),
 
475
        option.Option('quiet', 'Do not output what is being done.',
 
476
                      short_name='q'),
 
477
        option.Option('directory',
 
478
                      help='Branch to upload from, '
 
479
                      'rather than the one containing the working directory.',
 
480
                      short_name='d',
 
481
                      type=str,
 
482
                      ),
 
483
        option.Option('auto',
 
484
                      'Trigger an upload from this branch whenever the tip '
 
485
                      'revision changes.')
 
486
        ]
 
487
 
 
488
    def run(self, location=None, full=False, revision=None, remember=None,
 
489
            directory=None, quiet=False, auto=None, overwrite=False
 
490
            ):
 
491
        if directory is None:
 
492
            directory = u'.'
 
493
 
 
494
        (wt, branch,
 
495
         relpath) = controldir.ControlDir.open_containing_tree_or_branch(
 
496
             directory)
 
497
 
 
498
        if wt:
 
499
            locked = wt
 
500
        else:
 
501
            locked = branch
 
502
        with locked.lock_read():
 
503
            if wt:
 
504
                changes = wt.changes_from(wt.basis_tree())
 
505
 
 
506
                if revision is None and changes.has_changed():
 
507
                    raise errors.UncommittedChanges(wt)
 
508
 
 
509
            conf = branch.get_config_stack()
 
510
            if location is None:
 
511
                stored_loc = conf.get('upload_location')
 
512
                if stored_loc is None:
 
513
                    raise errors.CommandError(
 
514
                        'No upload location known or specified.')
 
515
                else:
 
516
                    # FIXME: Not currently tested
 
517
                    display_url = urlutils.unescape_for_display(stored_loc,
 
518
                                                                self.outf.encoding)
 
519
                    self.outf.write("Using saved location: %s\n" % display_url)
 
520
                    location = stored_loc
 
521
 
 
522
            to_transport = transport.get_transport(location)
 
523
 
 
524
            # Check that we are not uploading to a existing working tree.
 
525
            try:
 
526
                to_bzr_dir = controldir.ControlDir.open_from_transport(
 
527
                    to_transport)
 
528
                has_wt = to_bzr_dir.has_workingtree()
 
529
            except errors.NotBranchError:
 
530
                has_wt = False
 
531
            except errors.NotLocalUrl:
 
532
                # The exception raised is a bit weird... but that's life.
 
533
                has_wt = True
 
534
 
 
535
            if has_wt:
 
536
                raise CannotUploadToWorkingTree(url=location)
 
537
            if revision is None:
 
538
                rev_id = branch.last_revision()
 
539
            else:
 
540
                if len(revision) != 1:
 
541
                    raise errors.CommandError(
 
542
                        'bzr upload --revision takes exactly 1 argument')
 
543
                rev_id = revision[0].in_history(branch).rev_id
 
544
 
 
545
            tree = branch.repository.revision_tree(rev_id)
 
546
 
 
547
            uploader = BzrUploader(branch, to_transport, self.outf, tree,
 
548
                                   rev_id, quiet=quiet)
 
549
 
 
550
            if not overwrite:
 
551
                prev_uploaded_rev_id = uploader.get_uploaded_revid()
 
552
                graph = branch.repository.get_graph()
 
553
                if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
 
554
                    raise DivergedUploadedTree(
 
555
                        revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
 
556
            if full:
 
557
                uploader.upload_full_tree()
 
558
            else:
 
559
                uploader.upload_tree()
 
560
 
 
561
        # We uploaded successfully, remember it
 
562
        with branch.lock_write():
 
563
            upload_location = conf.get('upload_location')
 
564
            if upload_location is None or remember:
 
565
                conf.set('upload_location',
 
566
                         urlutils.unescape(to_transport.base))
 
567
            if auto is not None:
 
568
                conf.set('upload_auto', auto)