/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/plugins/upload/cmds.py

  • Committer: Breezy landing bot
  • Author(s): Jelmer Vernooij
  • Date: 2018-09-22 02:38:35 UTC
  • mfrom: (7058.5.4 upload-symlin)
  • Revision ID: breezy.the.bot@gmail.com-20180922023835-wb9nczxp63jpeudb
Add symlink support to 'brz upload'.

Merged from https://code.launchpad.net/~jelmer/brz/upload-symlinks/+merge/355061

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/plugins/upload/cmds.py
 
1
# Copyright (C) 2011, 2012 Canonical Ltd
 
2
#
 
3
# This program is free software; you can redistribute it and/or modify
 
4
# it under the terms of the GNU General Public License as published by
 
5
# the Free Software Foundation; either version 2 of the License, or
 
6
# (at your option) any later version.
 
7
#
 
8
# This program is distributed in the hope that it will be useful,
 
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
11
# GNU General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
16
 
 
17
"""bzr-upload command implementations."""
 
18
 
 
19
from __future__ import absolute_import
 
20
 
 
21
from ... import (
 
22
    branch,
 
23
    commands,
 
24
    config,
 
25
    lazy_import,
 
26
    option,
 
27
    osutils,
 
28
    )
 
29
lazy_import.lazy_import(globals(), """
 
30
import stat
 
31
import sys
 
32
 
 
33
from breezy import (
 
34
    controldir,
 
35
    errors,
 
36
    globbing,
 
37
    ignores,
 
38
    revision,
 
39
    transport,
 
40
    urlutils,
 
41
    )
 
42
""")
 
43
 
 
44
from ...sixish import (
 
45
    text_type,
 
46
    )
 
47
 
 
48
auto_option = config.Option(
 
49
    'upload_auto', default=False, from_unicode=config.bool_from_store,
 
50
    help="""\
 
51
Whether upload should occur when the tip of the branch changes.
 
52
""")
 
53
auto_quiet_option = config.Option(
 
54
    'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
 
55
    help="""\
 
56
Whether upload should occur quietly.
 
57
""")
 
58
location_option = config.Option(
 
59
    'upload_location', default=None,
 
60
    help="""\
 
61
The url to upload the working tree to.
 
62
""")
 
63
revid_location_option = config.Option(
 
64
    'upload_revid_location', default=u'.bzr-upload.revid',
 
65
    help="""\
 
66
The relative path to be used to store the uploaded revid.
 
67
 
 
68
The only bzr-related info uploaded with the working tree is the corresponding
 
69
revision id. The uploaded working tree is not linked to any other bzr data.
 
70
 
 
71
If the layout of your remote server is such that you can't write in the
 
72
root directory but only in the directories inside that root, you will need
 
73
to use the 'upload_revid_location' configuration variable to specify the
 
74
relative path to be used. That configuration variable can be specified in
 
75
locations.conf or branch.conf.
 
76
 
 
77
For example, given the following layout:
 
78
 
 
79
  Project/
 
80
    private/
 
81
    public/
 
82
 
 
83
you may have write access in 'private' and 'public' but in 'Project'
 
84
itself. In that case, you can add the following in your locations.conf or
 
85
branch.conf file:
 
86
 
 
87
  upload_revid_location = private/.bzr-upload.revid
 
88
""")
 
89
 
 
90
 
 
91
# FIXME: Add more tests around invalid paths or relative paths that doesn't
 
92
# exist on remote (if only to get proper error messages) for
 
93
# 'upload_revid_location'
 
94
 
 
95
 
 
96
class BzrUploader(object):
 
97
 
 
98
    def __init__(self, branch, to_transport, outf, tree, rev_id,
 
99
                 quiet=False):
 
100
        self.branch = branch
 
101
        self.to_transport = to_transport
 
102
        self.outf = outf
 
103
        self.tree = tree
 
104
        self.rev_id = rev_id
 
105
        self.quiet = quiet
 
106
        self._pending_deletions = []
 
107
        self._pending_renames = []
 
108
        self._uploaded_revid = None
 
109
        self._ignored = None
 
110
 
 
111
    def _up_stat(self, relpath):
 
112
        return self.to_transport.stat(urlutils.escape(relpath))
 
113
 
 
114
    def _up_rename(self, old_path, new_path):
 
115
        return self.to_transport.rename(urlutils.escape(old_path),
 
116
                                        urlutils.escape(new_path))
 
117
 
 
118
    def _up_delete(self, relpath):
 
119
        return self.to_transport.delete(urlutils.escape(relpath))
 
120
 
 
121
    def _up_delete_tree(self, relpath):
 
122
        return self.to_transport.delete_tree(urlutils.escape(relpath))
 
123
 
 
124
    def _up_mkdir(self, relpath, mode):
 
125
        return self.to_transport.mkdir(urlutils.escape(relpath), mode)
 
126
 
 
127
    def _up_rmdir(self, relpath):
 
128
        return self.to_transport.rmdir(urlutils.escape(relpath))
 
129
 
 
130
    def _up_put_bytes(self, relpath, bytes, mode):
 
131
        self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
 
132
 
 
133
    def _up_get_bytes(self, relpath):
 
134
        return self.to_transport.get_bytes(urlutils.escape(relpath))
 
135
 
 
136
    def set_uploaded_revid(self, rev_id):
 
137
        # XXX: Add tests for concurrent updates, etc.
 
138
        revid_path = self.branch.get_config_stack().get('upload_revid_location')
 
139
        self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
 
140
        self._uploaded_revid = rev_id
 
141
 
 
142
    def get_uploaded_revid(self):
 
143
        if self._uploaded_revid is None:
 
144
            revid_path = self.branch.get_config_stack(
 
145
                ).get('upload_revid_location')
 
146
            try:
 
147
                self._uploaded_revid = self._up_get_bytes(revid_path)
 
148
            except errors.NoSuchFile:
 
149
                # We have not uploaded to here.
 
150
                self._uploaded_revid = revision.NULL_REVISION
 
151
        return self._uploaded_revid
 
152
 
 
153
    def _get_ignored(self):
 
154
        if self._ignored is None:
 
155
            try:
 
156
                ignore_file_path = '.bzrignore-upload'
 
157
                ignore_file = self.tree.get_file(ignore_file_path)
 
158
            except errors.NoSuchFile:
 
159
                ignored_patterns = []
 
160
            else:
 
161
                ignored_patterns = ignores.parse_ignore_file(ignore_file)
 
162
            self._ignored = globbing.Globster(ignored_patterns)
 
163
        return self._ignored
 
164
 
 
165
    def is_ignored(self, relpath):
 
166
        glob = self._get_ignored()
 
167
        ignored = glob.match(relpath)
 
168
        import os
 
169
        if not ignored:
 
170
            # We still need to check that all parents are not ignored
 
171
            dir = os.path.dirname(relpath)
 
172
            while dir and not ignored:
 
173
                ignored = glob.match(dir)
 
174
                if not ignored:
 
175
                    dir = os.path.dirname(dir)
 
176
        return ignored
 
177
 
 
178
    def upload_file(self, old_relpath, new_relpath, id, mode=None):
 
179
        if mode is None:
 
180
            if self.tree.is_executable(new_relpath, id):
 
181
                mode = 0o775
 
182
            else:
 
183
                mode = 0o664
 
184
        if not self.quiet:
 
185
            self.outf.write('Uploading %s\n' % old_relpath)
 
186
        self._up_put_bytes(old_relpath, self.tree.get_file_text(new_relpath, id), mode)
 
187
 
 
188
    def _force_clear(self, relpath):
 
189
        try:
 
190
            st = self._up_stat(relpath)
 
191
            if stat.S_ISDIR(st.st_mode):
 
192
                # A simple rmdir may not be enough
 
193
                if not self.quiet:
 
194
                    self.outf.write('Clearing %s/%s\n' % (
 
195
                            self.to_transport.external_url(), relpath))
 
196
                self._up_delete_tree(relpath)
 
197
            elif stat.S_ISLNK(st.st_mode):
 
198
                if not self.quiet:
 
199
                    self.outf.write('Clearing %s/%s\n' % (
 
200
                        self.to_transport.external_url(), relpath))
 
201
                self._up_delete(relpath)
 
202
        except errors.PathError:
 
203
            pass
 
204
 
 
205
    def upload_file_robustly(self, relpath, id, mode=None):
 
206
        """Upload a file, clearing the way on the remote side.
 
207
 
 
208
        When doing a full upload, it may happen that a directory exists where
 
209
        we want to put our file.
 
210
        """
 
211
        self._force_clear(relpath)
 
212
        self.upload_file(relpath, relpath, id, mode)
 
213
 
 
214
    def upload_symlink(self, relpath, target):
 
215
        self.to_transport.symlink(target, relpath)
 
216
 
 
217
    def upload_symlink_robustly(self, relpath, target):
 
218
        """Handle uploading symlinks.
 
219
        """
 
220
        self._force_clear(relpath)
 
221
        # Target might not be there at this time; dummy file should be
 
222
        # overwritten at some point, possibly by another upload.
 
223
        target = osutils.normpath(osutils.pathjoin(
 
224
            osutils.dirname(relpath),
 
225
            target)
 
226
        )
 
227
        self.upload_symlink(relpath, target)
 
228
 
 
229
    def make_remote_dir(self, relpath, mode=None):
 
230
        if mode is None:
 
231
            mode = 0o775
 
232
        self._up_mkdir(relpath, mode)
 
233
 
 
234
    def make_remote_dir_robustly(self, relpath, mode=None):
 
235
        """Create a remote directory, clearing the way on the remote side.
 
236
 
 
237
        When doing a full upload, it may happen that a file exists where we
 
238
        want to create our directory.
 
239
        """
 
240
        try:
 
241
            st = self._up_stat(relpath)
 
242
            if not stat.S_ISDIR(st.st_mode):
 
243
                if not self.quiet:
 
244
                    self.outf.write('Deleting %s/%s\n' % (
 
245
                            self.to_transport.external_url(), relpath))
 
246
                self._up_delete(relpath)
 
247
            else:
 
248
                # Ok the remote dir already exists, nothing to do
 
249
                return
 
250
        except errors.PathError:
 
251
            pass
 
252
        self.make_remote_dir(relpath, mode)
 
253
 
 
254
    def delete_remote_file(self, relpath):
 
255
        if not self.quiet:
 
256
            self.outf.write('Deleting %s\n' % relpath)
 
257
        self._up_delete(relpath)
 
258
 
 
259
    def delete_remote_dir(self, relpath):
 
260
        if not self.quiet:
 
261
            self.outf.write('Deleting %s\n' % relpath)
 
262
        self._up_rmdir(relpath)
 
263
        # XXX: Add a test where a subdir is ignored but we still want to
 
264
        # delete the dir -- vila 100106
 
265
 
 
266
    def delete_remote_dir_maybe(self, relpath):
 
267
        """Try to delete relpath, keeping failures to retry later."""
 
268
        try:
 
269
            self._up_rmdir(relpath)
 
270
        # any kind of PathError would be OK, though we normally expect
 
271
        # DirectoryNotEmpty
 
272
        except errors.PathError:
 
273
            self._pending_deletions.append(relpath)
 
274
 
 
275
    def finish_deletions(self):
 
276
        if self._pending_deletions:
 
277
            # Process the previously failed deletions in reverse order to
 
278
            # delete children before parents
 
279
            for relpath in reversed(self._pending_deletions):
 
280
                self._up_rmdir(relpath)
 
281
            # The following shouldn't be needed since we use it once per
 
282
            # upload, but better safe than sorry ;-)
 
283
            self._pending_deletions = []
 
284
 
 
285
    def rename_remote(self, old_relpath, new_relpath):
 
286
        """Rename a remote file or directory taking care of collisions.
 
287
 
 
288
        To avoid collisions during bulk renames, each renamed target is
 
289
        temporarily assigned a unique name. When all renames have been done,
 
290
        each target get its proper name.
 
291
        """
 
292
        # We generate a sufficiently random name to *assume* that
 
293
        # no collisions will occur and don't worry about it (nor
 
294
        # handle it).
 
295
        import os
 
296
        import random
 
297
        import time
 
298
 
 
299
        stamp = '.tmp.%.9f.%d.%d' % (time.time(),
 
300
                                     os.getpid(),
 
301
                                     random.randint(0, 0x7FFFFFFF))
 
302
        if not self.quiet:
 
303
            self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
 
304
        self._up_rename(old_relpath, stamp)
 
305
        self._pending_renames.append((stamp, new_relpath))
 
306
 
 
307
    def finish_renames(self):
 
308
        for (stamp, new_path) in self._pending_renames:
 
309
            self._up_rename(stamp, new_path)
 
310
        # The following shouldn't be needed since we use it once per upload,
 
311
        # but better safe than sorry ;-)
 
312
        self._pending_renames = []
 
313
 
 
314
    def upload_full_tree(self):
 
315
        self.to_transport.ensure_base() # XXX: Handle errors (add
 
316
                                        # --create-prefix option ?)
 
317
        with self.tree.lock_read():
 
318
            for relpath, ie in self.tree.iter_entries_by_dir():
 
319
                if relpath in ('', '.bzrignore', '.bzrignore-upload'):
 
320
                    # skip root ('')
 
321
                    # .bzrignore and .bzrignore-upload have no meaning outside
 
322
                    # a working tree so do not upload them
 
323
                    continue
 
324
                if self.is_ignored(relpath):
 
325
                    if not self.quiet:
 
326
                        self.outf.write('Ignoring %s\n' % relpath)
 
327
                    continue
 
328
                if ie.kind == 'file':
 
329
                    self.upload_file_robustly(relpath, ie.file_id)
 
330
                elif ie.kind == 'symlink':
 
331
                    try:
 
332
                        self.upload_symlink_robustly(relpath, ie.symlink_target)
 
333
                    except errors.TransportNotPossible:
 
334
                        if not self.quiet:
 
335
                            target = self.tree.path_content_summary(relpath)[3]
 
336
                            self.outf.write('Not uploading symlink %s -> %s\n'
 
337
                                            % (relpath, target))
 
338
                elif ie.kind == 'directory':
 
339
                    self.make_remote_dir_robustly(relpath)
 
340
                else:
 
341
                    raise NotImplementedError
 
342
            self.set_uploaded_revid(self.rev_id)
 
343
 
 
344
    def upload_tree(self):
 
345
        # If we can't find the revid file on the remote location, upload the
 
346
        # full tree instead
 
347
        rev_id = self.get_uploaded_revid()
 
348
 
 
349
        if rev_id == revision.NULL_REVISION:
 
350
            if not self.quiet:
 
351
                self.outf.write('No uploaded revision id found,'
 
352
                                ' switching to full upload\n')
 
353
            self.upload_full_tree()
 
354
            # We're done
 
355
            return
 
356
 
 
357
        # Check if the revision hasn't already been uploaded
 
358
        if rev_id == self.rev_id:
 
359
            if not self.quiet:
 
360
                self.outf.write('Remote location already up to date\n')
 
361
 
 
362
        from_tree = self.branch.repository.revision_tree(rev_id)
 
363
        self.to_transport.ensure_base() # XXX: Handle errors (add
 
364
                                        # --create-prefix option ?)
 
365
        changes = self.tree.changes_from(from_tree)
 
366
        with self.tree.lock_read():
 
367
            for (path, id, kind) in changes.removed:
 
368
                if self.is_ignored(path):
 
369
                    if not self.quiet:
 
370
                        self.outf.write('Ignoring %s\n' % path)
 
371
                    continue
 
372
                if kind is 'file':
 
373
                    self.delete_remote_file(path)
 
374
                elif kind is  'directory':
 
375
                    self.delete_remote_dir_maybe(path)
 
376
                elif kind == 'symlink':
 
377
                    self.delete_remote_file(path)
 
378
                else:
 
379
                    raise NotImplementedError
 
380
 
 
381
            for (old_path, new_path, id, kind,
 
382
                 content_change, exec_change) in changes.renamed:
 
383
                if self.is_ignored(old_path) and self.is_ignored(new_path):
 
384
                    if not self.quiet:
 
385
                        self.outf.write('Ignoring %s\n' % old_path)
 
386
                        self.outf.write('Ignoring %s\n' % new_path)
 
387
                    continue
 
388
                if content_change:
 
389
                    # We update the old_path content because renames and
 
390
                    # deletions are differed.
 
391
                    self.upload_file(old_path, new_path, id)
 
392
                self.rename_remote(old_path, new_path)
 
393
            self.finish_renames()
 
394
            self.finish_deletions()
 
395
 
 
396
            for (path, id, old_kind, new_kind) in changes.kind_changed:
 
397
                if self.is_ignored(path):
 
398
                    if not self.quiet:
 
399
                        self.outf.write('Ignoring %s\n' % path)
 
400
                    continue
 
401
                if old_kind in ('file', 'symlink'):
 
402
                    self.delete_remote_file(path)
 
403
                elif old_kind == 'directory':
 
404
                    self.delete_remote_dir(path)
 
405
                else:
 
406
                    raise NotImplementedError
 
407
 
 
408
                if new_kind == 'file':
 
409
                    self.upload_file(path, path, id)
 
410
                elif new_kind == 'symlink':
 
411
                    target = self.tree.get_symlink_target(path)
 
412
                    self.upload_symlink(path, target)
 
413
                elif new_kind is 'directory':
 
414
                    self.make_remote_dir(path)
 
415
                else:
 
416
                    raise NotImplementedError
 
417
 
 
418
            for (path, id, kind) in changes.added:
 
419
                if self.is_ignored(path):
 
420
                    if not self.quiet:
 
421
                        self.outf.write('Ignoring %s\n' % path)
 
422
                    continue
 
423
                if kind == 'file':
 
424
                    self.upload_file(path, path, id)
 
425
                elif kind == 'directory':
 
426
                    self.make_remote_dir(path)
 
427
                elif kind == 'symlink':
 
428
                    target = self.tree.get_symlink_target(path)
 
429
                    try:
 
430
                        self.upload_symlink(path, target)
 
431
                    except errors.TransportNotPossible:
 
432
                        if not self.quiet:
 
433
                            self.outf.write('Not uploading symlink %s -> %s\n'
 
434
                                            % (path, target))
 
435
                else:
 
436
                    raise NotImplementedError
 
437
 
 
438
            # XXX: Add a test for exec_change
 
439
            for (path, id, kind,
 
440
                 content_change, exec_change) in changes.modified:
 
441
                if self.is_ignored(path):
 
442
                    if not self.quiet:
 
443
                        self.outf.write('Ignoring %s\n' % path)
 
444
                    continue
 
445
                if kind == 'file':
 
446
                    self.upload_file(path, path, id)
 
447
                elif kind == 'symlink':
 
448
                    target = self.tree.get_symlink_target(path)
 
449
                    self.upload_symlink(path, target)
 
450
                else:
 
451
                    raise NotImplementedError
 
452
 
 
453
            self.set_uploaded_revid(self.rev_id)
 
454
 
 
455
 
 
456
class CannotUploadToWorkingTree(errors.BzrCommandError):
 
457
 
 
458
    _fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
 
459
 
 
460
 
 
461
class DivergedUploadedTree(errors.BzrCommandError):
 
462
 
 
463
    _fmt = ("Your branch (%(revid)s)"
 
464
            " and the uploaded tree (%(uploaded_revid)s) have diverged: ")
 
465
 
 
466
 
 
467
class cmd_upload(commands.Command):
 
468
    """Upload a working tree, as a whole or incrementally.
 
469
 
 
470
    If no destination is specified use the last one used.
 
471
    If no revision is specified upload the changes since the last upload.
 
472
 
 
473
    Changes include files added, renamed, modified or removed.
 
474
    """
 
475
    _see_also = ['plugins/upload']
 
476
    takes_args = ['location?']
 
477
    takes_options = [
 
478
        'revision',
 
479
        'remember',
 
480
        'overwrite',
 
481
        option.Option('full', 'Upload the full working tree.'),
 
482
        option.Option('quiet', 'Do not output what is being done.',
 
483
                       short_name='q'),
 
484
        option.Option('directory',
 
485
                      help='Branch to upload from, '
 
486
                      'rather than the one containing the working directory.',
 
487
                      short_name='d',
 
488
                      type=text_type,
 
489
                      ),
 
490
        option.Option('auto',
 
491
                      'Trigger an upload from this branch whenever the tip '
 
492
                      'revision changes.')
 
493
       ]
 
494
 
 
495
    def run(self, location=None, full=False, revision=None, remember=None,
 
496
            directory=None, quiet=False, auto=None, overwrite=False
 
497
            ):
 
498
        if directory is None:
 
499
            directory = u'.'
 
500
 
 
501
        (wt, branch,
 
502
         relpath) = controldir.ControlDir.open_containing_tree_or_branch(
 
503
             directory)
 
504
 
 
505
        if wt:
 
506
            wt.lock_read()
 
507
            locked = wt
 
508
        else:
 
509
            branch.lock_read()
 
510
            locked = branch
 
511
        try:
 
512
            if wt:
 
513
                changes = wt.changes_from(wt.basis_tree())
 
514
 
 
515
                if revision is None and  changes.has_changed():
 
516
                    raise errors.UncommittedChanges(wt)
 
517
 
 
518
            conf = branch.get_config_stack()
 
519
            if location is None:
 
520
                stored_loc = conf.get('upload_location')
 
521
                if stored_loc is None:
 
522
                    raise errors.BzrCommandError(
 
523
                        'No upload location known or specified.')
 
524
                else:
 
525
                    # FIXME: Not currently tested
 
526
                    display_url = urlutils.unescape_for_display(stored_loc,
 
527
                            self.outf.encoding)
 
528
                    self.outf.write("Using saved location: %s\n" % display_url)
 
529
                    location = stored_loc
 
530
 
 
531
            to_transport = transport.get_transport(location)
 
532
 
 
533
            # Check that we are not uploading to a existing working tree.
 
534
            try:
 
535
                to_bzr_dir = controldir.ControlDir.open_from_transport(
 
536
                        to_transport)
 
537
                has_wt = to_bzr_dir.has_workingtree()
 
538
            except errors.NotBranchError:
 
539
                has_wt = False
 
540
            except errors.NotLocalUrl:
 
541
                # The exception raised is a bit weird... but that's life.
 
542
                has_wt = True
 
543
 
 
544
            if has_wt:
 
545
                raise CannotUploadToWorkingTree(url=location)
 
546
            if revision is None:
 
547
                rev_id = branch.last_revision()
 
548
            else:
 
549
                if len(revision) != 1:
 
550
                    raise errors.BzrCommandError(
 
551
                        'bzr upload --revision takes exactly 1 argument')
 
552
                rev_id = revision[0].in_history(branch).rev_id
 
553
 
 
554
            tree = branch.repository.revision_tree(rev_id)
 
555
 
 
556
            uploader = BzrUploader(branch, to_transport, self.outf, tree,
 
557
                                   rev_id, quiet=quiet)
 
558
 
 
559
            if not overwrite:
 
560
                prev_uploaded_rev_id = uploader.get_uploaded_revid()
 
561
                graph = branch.repository.get_graph()
 
562
                if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
 
563
                    raise DivergedUploadedTree(
 
564
                        revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
 
565
            if full:
 
566
                uploader.upload_full_tree()
 
567
            else:
 
568
                uploader.upload_tree()
 
569
        finally:
 
570
            locked.unlock()
 
571
 
 
572
        # We uploaded successfully, remember it
 
573
        with branch.lock_write():
 
574
            upload_location = conf.get('upload_location')
 
575
            if upload_location is None or remember:
 
576
                conf.set('upload_location',
 
577
                         urlutils.unescape(to_transport.base))
 
578
            if auto is not None:
 
579
                conf.set('upload_auto', auto)
 
580