/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/plugins/upload/cmds.py

  • Committer: Jelmer Vernooij
  • Date: 2019-02-15 17:41:17 UTC
  • mto: (7290.1.2 work)
  • mto: This revision was merged to the branch mainline in revision 7295.
  • Revision ID: jelmer@jelmer.uk-20190215174117-o9w1am2z88mg9g1q
Update references to home location.

~/.config/breezy rather than ~/.bazaar.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/plugins/upload/cmds.py
 
1
# Copyright (C) 2011, 2012 Canonical Ltd
 
2
#
 
3
# This program is free software; you can redistribute it and/or modify
 
4
# it under the terms of the GNU General Public License as published by
 
5
# the Free Software Foundation; either version 2 of the License, or
 
6
# (at your option) any later version.
 
7
#
 
8
# This program is distributed in the hope that it will be useful,
 
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
11
# GNU General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
16
 
 
17
"""bzr-upload command implementations."""
 
18
 
 
19
from __future__ import absolute_import
 
20
 
 
21
from ... import (
 
22
    commands,
 
23
    config,
 
24
    lazy_import,
 
25
    option,
 
26
    osutils,
 
27
    )
 
28
lazy_import.lazy_import(globals(), """
 
29
import stat
 
30
 
 
31
from breezy import (
 
32
    controldir,
 
33
    errors,
 
34
    globbing,
 
35
    ignores,
 
36
    revision,
 
37
    transport,
 
38
    urlutils,
 
39
    )
 
40
""")
 
41
 
 
42
from ...sixish import (
 
43
    text_type,
 
44
    )
 
45
 
 
46
auto_option = config.Option(
 
47
    'upload_auto', default=False, from_unicode=config.bool_from_store,
 
48
    help="""\
 
49
Whether upload should occur when the tip of the branch changes.
 
50
""")
 
51
auto_quiet_option = config.Option(
 
52
    'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
 
53
    help="""\
 
54
Whether upload should occur quietly.
 
55
""")
 
56
location_option = config.Option(
 
57
    'upload_location', default=None,
 
58
    help="""\
 
59
The url to upload the working tree to.
 
60
""")
 
61
revid_location_option = config.Option(
 
62
    'upload_revid_location', default=u'.bzr-upload.revid',
 
63
    help="""\
 
64
The relative path to be used to store the uploaded revid.
 
65
 
 
66
The only bzr-related info uploaded with the working tree is the corresponding
 
67
revision id. The uploaded working tree is not linked to any other bzr data.
 
68
 
 
69
If the layout of your remote server is such that you can't write in the
 
70
root directory but only in the directories inside that root, you will need
 
71
to use the 'upload_revid_location' configuration variable to specify the
 
72
relative path to be used. That configuration variable can be specified in
 
73
locations.conf or branch.conf.
 
74
 
 
75
For example, given the following layout:
 
76
 
 
77
  Project/
 
78
    private/
 
79
    public/
 
80
 
 
81
you may have write access in 'private' and 'public' but in 'Project'
 
82
itself. In that case, you can add the following in your locations.conf or
 
83
branch.conf file:
 
84
 
 
85
  upload_revid_location = private/.bzr-upload.revid
 
86
""")
 
87
 
 
88
 
 
89
# FIXME: Add more tests around invalid paths or relative paths that doesn't
 
90
# exist on remote (if only to get proper error messages) for
 
91
# 'upload_revid_location'
 
92
 
 
93
 
 
94
class BzrUploader(object):
 
95
 
 
96
    def __init__(self, branch, to_transport, outf, tree, rev_id,
 
97
                 quiet=False):
 
98
        self.branch = branch
 
99
        self.to_transport = to_transport
 
100
        self.outf = outf
 
101
        self.tree = tree
 
102
        self.rev_id = rev_id
 
103
        self.quiet = quiet
 
104
        self._pending_deletions = []
 
105
        self._pending_renames = []
 
106
        self._uploaded_revid = None
 
107
        self._ignored = None
 
108
 
 
109
    def _up_stat(self, relpath):
 
110
        return self.to_transport.stat(urlutils.escape(relpath))
 
111
 
 
112
    def _up_rename(self, old_path, new_path):
 
113
        return self.to_transport.rename(urlutils.escape(old_path),
 
114
                                        urlutils.escape(new_path))
 
115
 
 
116
    def _up_delete(self, relpath):
 
117
        return self.to_transport.delete(urlutils.escape(relpath))
 
118
 
 
119
    def _up_delete_tree(self, relpath):
 
120
        return self.to_transport.delete_tree(urlutils.escape(relpath))
 
121
 
 
122
    def _up_mkdir(self, relpath, mode):
 
123
        return self.to_transport.mkdir(urlutils.escape(relpath), mode)
 
124
 
 
125
    def _up_rmdir(self, relpath):
 
126
        return self.to_transport.rmdir(urlutils.escape(relpath))
 
127
 
 
128
    def _up_put_bytes(self, relpath, bytes, mode):
 
129
        self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
 
130
 
 
131
    def _up_get_bytes(self, relpath):
 
132
        return self.to_transport.get_bytes(urlutils.escape(relpath))
 
133
 
 
134
    def set_uploaded_revid(self, rev_id):
 
135
        # XXX: Add tests for concurrent updates, etc.
 
136
        revid_path = self.branch.get_config_stack().get('upload_revid_location')
 
137
        self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
 
138
        self._uploaded_revid = rev_id
 
139
 
 
140
    def get_uploaded_revid(self):
 
141
        if self._uploaded_revid is None:
 
142
            revid_path = self.branch.get_config_stack(
 
143
                ).get('upload_revid_location')
 
144
            try:
 
145
                self._uploaded_revid = self._up_get_bytes(revid_path)
 
146
            except errors.NoSuchFile:
 
147
                # We have not uploaded to here.
 
148
                self._uploaded_revid = revision.NULL_REVISION
 
149
        return self._uploaded_revid
 
150
 
 
151
    def _get_ignored(self):
 
152
        if self._ignored is None:
 
153
            try:
 
154
                ignore_file_path = '.bzrignore-upload'
 
155
                ignore_file = self.tree.get_file(ignore_file_path)
 
156
            except errors.NoSuchFile:
 
157
                ignored_patterns = []
 
158
            else:
 
159
                ignored_patterns = ignores.parse_ignore_file(ignore_file)
 
160
            self._ignored = globbing.Globster(ignored_patterns)
 
161
        return self._ignored
 
162
 
 
163
    def is_ignored(self, relpath):
 
164
        glob = self._get_ignored()
 
165
        ignored = glob.match(relpath)
 
166
        import os
 
167
        if not ignored:
 
168
            # We still need to check that all parents are not ignored
 
169
            dir = os.path.dirname(relpath)
 
170
            while dir and not ignored:
 
171
                ignored = glob.match(dir)
 
172
                if not ignored:
 
173
                    dir = os.path.dirname(dir)
 
174
        return ignored
 
175
 
 
176
    def upload_file(self, old_relpath, new_relpath, mode=None):
 
177
        if mode is None:
 
178
            if self.tree.is_executable(new_relpath):
 
179
                mode = 0o775
 
180
            else:
 
181
                mode = 0o664
 
182
        if not self.quiet:
 
183
            self.outf.write('Uploading %s\n' % old_relpath)
 
184
        self._up_put_bytes(
 
185
            old_relpath, self.tree.get_file_text(new_relpath), mode)
 
186
 
 
187
    def _force_clear(self, relpath):
 
188
        try:
 
189
            st = self._up_stat(relpath)
 
190
            if stat.S_ISDIR(st.st_mode):
 
191
                # A simple rmdir may not be enough
 
192
                if not self.quiet:
 
193
                    self.outf.write('Clearing %s/%s\n' % (
 
194
                        self.to_transport.external_url(), relpath))
 
195
                self._up_delete_tree(relpath)
 
196
            elif stat.S_ISLNK(st.st_mode):
 
197
                if not self.quiet:
 
198
                    self.outf.write('Clearing %s/%s\n' % (
 
199
                        self.to_transport.external_url(), relpath))
 
200
                self._up_delete(relpath)
 
201
        except errors.PathError:
 
202
            pass
 
203
 
 
204
    def upload_file_robustly(self, relpath, mode=None):
 
205
        """Upload a file, clearing the way on the remote side.
 
206
 
 
207
        When doing a full upload, it may happen that a directory exists where
 
208
        we want to put our file.
 
209
        """
 
210
        self._force_clear(relpath)
 
211
        self.upload_file(relpath, relpath, mode)
 
212
 
 
213
    def upload_symlink(self, relpath, target):
 
214
        self.to_transport.symlink(target, relpath)
 
215
 
 
216
    def upload_symlink_robustly(self, relpath, target):
 
217
        """Handle uploading symlinks.
 
218
        """
 
219
        self._force_clear(relpath)
 
220
        # Target might not be there at this time; dummy file should be
 
221
        # overwritten at some point, possibly by another upload.
 
222
        target = osutils.normpath(osutils.pathjoin(
 
223
            osutils.dirname(relpath),
 
224
            target)
 
225
        )
 
226
        self.upload_symlink(relpath, target)
 
227
 
 
228
    def make_remote_dir(self, relpath, mode=None):
 
229
        if mode is None:
 
230
            mode = 0o775
 
231
        self._up_mkdir(relpath, mode)
 
232
 
 
233
    def make_remote_dir_robustly(self, relpath, mode=None):
 
234
        """Create a remote directory, clearing the way on the remote side.
 
235
 
 
236
        When doing a full upload, it may happen that a file exists where we
 
237
        want to create our directory.
 
238
        """
 
239
        try:
 
240
            st = self._up_stat(relpath)
 
241
            if not stat.S_ISDIR(st.st_mode):
 
242
                if not self.quiet:
 
243
                    self.outf.write('Deleting %s/%s\n' % (
 
244
                        self.to_transport.external_url(), relpath))
 
245
                self._up_delete(relpath)
 
246
            else:
 
247
                # Ok the remote dir already exists, nothing to do
 
248
                return
 
249
        except errors.PathError:
 
250
            pass
 
251
        self.make_remote_dir(relpath, mode)
 
252
 
 
253
    def delete_remote_file(self, relpath):
 
254
        if not self.quiet:
 
255
            self.outf.write('Deleting %s\n' % relpath)
 
256
        self._up_delete(relpath)
 
257
 
 
258
    def delete_remote_dir(self, relpath):
 
259
        if not self.quiet:
 
260
            self.outf.write('Deleting %s\n' % relpath)
 
261
        self._up_rmdir(relpath)
 
262
        # XXX: Add a test where a subdir is ignored but we still want to
 
263
        # delete the dir -- vila 100106
 
264
 
 
265
    def delete_remote_dir_maybe(self, relpath):
 
266
        """Try to delete relpath, keeping failures to retry later."""
 
267
        try:
 
268
            self._up_rmdir(relpath)
 
269
        # any kind of PathError would be OK, though we normally expect
 
270
        # DirectoryNotEmpty
 
271
        except errors.PathError:
 
272
            self._pending_deletions.append(relpath)
 
273
 
 
274
    def finish_deletions(self):
 
275
        if self._pending_deletions:
 
276
            # Process the previously failed deletions in reverse order to
 
277
            # delete children before parents
 
278
            for relpath in reversed(self._pending_deletions):
 
279
                self._up_rmdir(relpath)
 
280
            # The following shouldn't be needed since we use it once per
 
281
            # upload, but better safe than sorry ;-)
 
282
            self._pending_deletions = []
 
283
 
 
284
    def rename_remote(self, old_relpath, new_relpath):
 
285
        """Rename a remote file or directory taking care of collisions.
 
286
 
 
287
        To avoid collisions during bulk renames, each renamed target is
 
288
        temporarily assigned a unique name. When all renames have been done,
 
289
        each target get its proper name.
 
290
        """
 
291
        # We generate a sufficiently random name to *assume* that
 
292
        # no collisions will occur and don't worry about it (nor
 
293
        # handle it).
 
294
        import os
 
295
        import random
 
296
        import time
 
297
 
 
298
        stamp = '.tmp.%.9f.%d.%d' % (time.time(),
 
299
                                     os.getpid(),
 
300
                                     random.randint(0, 0x7FFFFFFF))
 
301
        if not self.quiet:
 
302
            self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
 
303
        self._up_rename(old_relpath, stamp)
 
304
        self._pending_renames.append((stamp, new_relpath))
 
305
 
 
306
    def finish_renames(self):
 
307
        for (stamp, new_path) in self._pending_renames:
 
308
            self._up_rename(stamp, new_path)
 
309
        # The following shouldn't be needed since we use it once per upload,
 
310
        # but better safe than sorry ;-)
 
311
        self._pending_renames = []
 
312
 
 
313
    def upload_full_tree(self):
 
314
        self.to_transport.ensure_base()  # XXX: Handle errors (add
 
315
        # --create-prefix option ?)
 
316
        with self.tree.lock_read():
 
317
            for relpath, ie in self.tree.iter_entries_by_dir():
 
318
                if relpath in ('', '.bzrignore', '.bzrignore-upload'):
 
319
                    # skip root ('')
 
320
                    # .bzrignore and .bzrignore-upload have no meaning outside
 
321
                    # a working tree so do not upload them
 
322
                    continue
 
323
                if self.is_ignored(relpath):
 
324
                    if not self.quiet:
 
325
                        self.outf.write('Ignoring %s\n' % relpath)
 
326
                    continue
 
327
                if ie.kind == 'file':
 
328
                    self.upload_file_robustly(relpath)
 
329
                elif ie.kind == 'symlink':
 
330
                    try:
 
331
                        self.upload_symlink_robustly(
 
332
                            relpath, ie.symlink_target)
 
333
                    except errors.TransportNotPossible:
 
334
                        if not self.quiet:
 
335
                            target = self.tree.path_content_summary(relpath)[3]
 
336
                            self.outf.write('Not uploading symlink %s -> %s\n'
 
337
                                            % (relpath, target))
 
338
                elif ie.kind == 'directory':
 
339
                    self.make_remote_dir_robustly(relpath)
 
340
                else:
 
341
                    raise NotImplementedError
 
342
            self.set_uploaded_revid(self.rev_id)
 
343
 
 
344
    def upload_tree(self):
 
345
        # If we can't find the revid file on the remote location, upload the
 
346
        # full tree instead
 
347
        rev_id = self.get_uploaded_revid()
 
348
 
 
349
        if rev_id == revision.NULL_REVISION:
 
350
            if not self.quiet:
 
351
                self.outf.write('No uploaded revision id found,'
 
352
                                ' switching to full upload\n')
 
353
            self.upload_full_tree()
 
354
            # We're done
 
355
            return
 
356
 
 
357
        # Check if the revision hasn't already been uploaded
 
358
        if rev_id == self.rev_id:
 
359
            if not self.quiet:
 
360
                self.outf.write('Remote location already up to date\n')
 
361
 
 
362
        from_tree = self.branch.repository.revision_tree(rev_id)
 
363
        self.to_transport.ensure_base()  # XXX: Handle errors (add
 
364
        # --create-prefix option ?)
 
365
        changes = self.tree.changes_from(from_tree)
 
366
        with self.tree.lock_read():
 
367
            for (path, id, kind) in changes.removed:
 
368
                if self.is_ignored(path):
 
369
                    if not self.quiet:
 
370
                        self.outf.write('Ignoring %s\n' % path)
 
371
                    continue
 
372
                if kind == 'file':
 
373
                    self.delete_remote_file(path)
 
374
                elif kind == 'directory':
 
375
                    self.delete_remote_dir_maybe(path)
 
376
                elif kind == 'symlink':
 
377
                    self.delete_remote_file(path)
 
378
                else:
 
379
                    raise NotImplementedError
 
380
 
 
381
            for (old_path, new_path, id, kind,
 
382
                 content_change, exec_change) in changes.renamed:
 
383
                if self.is_ignored(old_path) and self.is_ignored(new_path):
 
384
                    if not self.quiet:
 
385
                        self.outf.write('Ignoring %s\n' % old_path)
 
386
                        self.outf.write('Ignoring %s\n' % new_path)
 
387
                    continue
 
388
                if content_change:
 
389
                    # We update the old_path content because renames and
 
390
                    # deletions are differed.
 
391
                    self.upload_file(old_path, new_path)
 
392
                self.rename_remote(old_path, new_path)
 
393
            self.finish_renames()
 
394
            self.finish_deletions()
 
395
 
 
396
            for (path, id, old_kind, new_kind) in changes.kind_changed:
 
397
                if self.is_ignored(path):
 
398
                    if not self.quiet:
 
399
                        self.outf.write('Ignoring %s\n' % path)
 
400
                    continue
 
401
                if old_kind in ('file', 'symlink'):
 
402
                    self.delete_remote_file(path)
 
403
                elif old_kind == 'directory':
 
404
                    self.delete_remote_dir(path)
 
405
                else:
 
406
                    raise NotImplementedError
 
407
 
 
408
                if new_kind == 'file':
 
409
                    self.upload_file(path, path)
 
410
                elif new_kind == 'symlink':
 
411
                    target = self.tree.get_symlink_target(path)
 
412
                    self.upload_symlink(path, target)
 
413
                elif new_kind == 'directory':
 
414
                    self.make_remote_dir(path)
 
415
                else:
 
416
                    raise NotImplementedError
 
417
 
 
418
            for (path, id, kind) in changes.added:
 
419
                if self.is_ignored(path):
 
420
                    if not self.quiet:
 
421
                        self.outf.write('Ignoring %s\n' % path)
 
422
                    continue
 
423
                if kind == 'file':
 
424
                    self.upload_file(path, path)
 
425
                elif kind == 'directory':
 
426
                    self.make_remote_dir(path)
 
427
                elif kind == 'symlink':
 
428
                    target = self.tree.get_symlink_target(path)
 
429
                    try:
 
430
                        self.upload_symlink(path, target)
 
431
                    except errors.TransportNotPossible:
 
432
                        if not self.quiet:
 
433
                            self.outf.write('Not uploading symlink %s -> %s\n'
 
434
                                            % (path, target))
 
435
                else:
 
436
                    raise NotImplementedError
 
437
 
 
438
            # XXX: Add a test for exec_change
 
439
            for (path, id, kind,
 
440
                 content_change, exec_change) in changes.modified:
 
441
                if self.is_ignored(path):
 
442
                    if not self.quiet:
 
443
                        self.outf.write('Ignoring %s\n' % path)
 
444
                    continue
 
445
                if kind == 'file':
 
446
                    self.upload_file(path, path)
 
447
                elif kind == 'symlink':
 
448
                    target = self.tree.get_symlink_target(path)
 
449
                    self.upload_symlink(path, target)
 
450
                else:
 
451
                    raise NotImplementedError
 
452
 
 
453
            self.set_uploaded_revid(self.rev_id)
 
454
 
 
455
 
 
456
class CannotUploadToWorkingTree(errors.BzrCommandError):
 
457
 
 
458
    _fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
 
459
 
 
460
 
 
461
class DivergedUploadedTree(errors.BzrCommandError):
 
462
 
 
463
    _fmt = ("Your branch (%(revid)s)"
 
464
            " and the uploaded tree (%(uploaded_revid)s) have diverged: ")
 
465
 
 
466
 
 
467
class cmd_upload(commands.Command):
 
468
    """Upload a working tree, as a whole or incrementally.
 
469
 
 
470
    If no destination is specified use the last one used.
 
471
    If no revision is specified upload the changes since the last upload.
 
472
 
 
473
    Changes include files added, renamed, modified or removed.
 
474
    """
 
475
    _see_also = ['plugins/upload']
 
476
    takes_args = ['location?']
 
477
    takes_options = [
 
478
        'revision',
 
479
        'remember',
 
480
        'overwrite',
 
481
        option.Option('full', 'Upload the full working tree.'),
 
482
        option.Option('quiet', 'Do not output what is being done.',
 
483
                      short_name='q'),
 
484
        option.Option('directory',
 
485
                      help='Branch to upload from, '
 
486
                      'rather than the one containing the working directory.',
 
487
                      short_name='d',
 
488
                      type=text_type,
 
489
                      ),
 
490
        option.Option('auto',
 
491
                      'Trigger an upload from this branch whenever the tip '
 
492
                      'revision changes.')
 
493
        ]
 
494
 
 
495
    def run(self, location=None, full=False, revision=None, remember=None,
 
496
            directory=None, quiet=False, auto=None, overwrite=False
 
497
            ):
 
498
        if directory is None:
 
499
            directory = u'.'
 
500
 
 
501
        (wt, branch,
 
502
         relpath) = controldir.ControlDir.open_containing_tree_or_branch(
 
503
             directory)
 
504
 
 
505
        if wt:
 
506
            wt.lock_read()
 
507
            locked = wt
 
508
        else:
 
509
            branch.lock_read()
 
510
            locked = branch
 
511
        try:
 
512
            if wt:
 
513
                changes = wt.changes_from(wt.basis_tree())
 
514
 
 
515
                if revision is None and changes.has_changed():
 
516
                    raise errors.UncommittedChanges(wt)
 
517
 
 
518
            conf = branch.get_config_stack()
 
519
            if location is None:
 
520
                stored_loc = conf.get('upload_location')
 
521
                if stored_loc is None:
 
522
                    raise errors.BzrCommandError(
 
523
                        'No upload location known or specified.')
 
524
                else:
 
525
                    # FIXME: Not currently tested
 
526
                    display_url = urlutils.unescape_for_display(stored_loc,
 
527
                                                                self.outf.encoding)
 
528
                    self.outf.write("Using saved location: %s\n" % display_url)
 
529
                    location = stored_loc
 
530
 
 
531
            to_transport = transport.get_transport(location)
 
532
 
 
533
            # Check that we are not uploading to a existing working tree.
 
534
            try:
 
535
                to_bzr_dir = controldir.ControlDir.open_from_transport(
 
536
                    to_transport)
 
537
                has_wt = to_bzr_dir.has_workingtree()
 
538
            except errors.NotBranchError:
 
539
                has_wt = False
 
540
            except errors.NotLocalUrl:
 
541
                # The exception raised is a bit weird... but that's life.
 
542
                has_wt = True
 
543
 
 
544
            if has_wt:
 
545
                raise CannotUploadToWorkingTree(url=location)
 
546
            if revision is None:
 
547
                rev_id = branch.last_revision()
 
548
            else:
 
549
                if len(revision) != 1:
 
550
                    raise errors.BzrCommandError(
 
551
                        'bzr upload --revision takes exactly 1 argument')
 
552
                rev_id = revision[0].in_history(branch).rev_id
 
553
 
 
554
            tree = branch.repository.revision_tree(rev_id)
 
555
 
 
556
            uploader = BzrUploader(branch, to_transport, self.outf, tree,
 
557
                                   rev_id, quiet=quiet)
 
558
 
 
559
            if not overwrite:
 
560
                prev_uploaded_rev_id = uploader.get_uploaded_revid()
 
561
                graph = branch.repository.get_graph()
 
562
                if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
 
563
                    raise DivergedUploadedTree(
 
564
                        revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
 
565
            if full:
 
566
                uploader.upload_full_tree()
 
567
            else:
 
568
                uploader.upload_tree()
 
569
        finally:
 
570
            locked.unlock()
 
571
 
 
572
        # We uploaded successfully, remember it
 
573
        with branch.lock_write():
 
574
            upload_location = conf.get('upload_location')
 
575
            if upload_location is None or remember:
 
576
                conf.set('upload_location',
 
577
                         urlutils.unescape(to_transport.base))
 
578
            if auto is not None:
 
579
                conf.set('upload_auto', auto)