/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/plugins/upload/cmds.py

  • Committer: Jelmer Vernooij
  • Date: 2017-05-21 12:41:27 UTC
  • mto: This revision was merged to the branch mainline in revision 6623.
  • Revision ID: jelmer@jelmer.uk-20170521124127-iv8etg0vwymyai6y
s/bzr/brz/ in apport config.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/plugins/upload/cmds.py
1
 
# Copyright (C) 2011, 2012 Canonical Ltd
2
 
#
3
 
# This program is free software; you can redistribute it and/or modify
4
 
# it under the terms of the GNU General Public License as published by
5
 
# the Free Software Foundation; either version 2 of the License, or
6
 
# (at your option) any later version.
7
 
#
8
 
# This program is distributed in the hope that it will be useful,
9
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
 
# GNU General Public License for more details.
12
 
#
13
 
# You should have received a copy of the GNU General Public License
14
 
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
 
 
17
 
"""bzr-upload command implementations."""
18
 
 
19
 
from ... import (
20
 
    commands,
21
 
    config,
22
 
    errors,
23
 
    lazy_import,
24
 
    option,
25
 
    osutils,
26
 
    )
27
 
lazy_import.lazy_import(globals(), """
28
 
import stat
29
 
 
30
 
from breezy import (
31
 
    controldir,
32
 
    globbing,
33
 
    ignores,
34
 
    revision,
35
 
    transport,
36
 
    urlutils,
37
 
    )
38
 
""")
39
 
 
40
 
 
41
 
auto_option = config.Option(
42
 
    'upload_auto', default=False, from_unicode=config.bool_from_store,
43
 
    help="""\
44
 
Whether upload should occur when the tip of the branch changes.
45
 
""")
46
 
auto_quiet_option = config.Option(
47
 
    'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
48
 
    help="""\
49
 
Whether upload should occur quietly.
50
 
""")
51
 
location_option = config.Option(
52
 
    'upload_location', default=None,
53
 
    help="""\
54
 
The url to upload the working tree to.
55
 
""")
56
 
revid_location_option = config.Option(
57
 
    'upload_revid_location', default=u'.bzr-upload.revid',
58
 
    help="""\
59
 
The relative path to be used to store the uploaded revid.
60
 
 
61
 
The only bzr-related info uploaded with the working tree is the corresponding
62
 
revision id. The uploaded working tree is not linked to any other bzr data.
63
 
 
64
 
If the layout of your remote server is such that you can't write in the
65
 
root directory but only in the directories inside that root, you will need
66
 
to use the 'upload_revid_location' configuration variable to specify the
67
 
relative path to be used. That configuration variable can be specified in
68
 
locations.conf or branch.conf.
69
 
 
70
 
For example, given the following layout:
71
 
 
72
 
  Project/
73
 
    private/
74
 
    public/
75
 
 
76
 
you may have write access in 'private' and 'public' but in 'Project'
77
 
itself. In that case, you can add the following in your locations.conf or
78
 
branch.conf file:
79
 
 
80
 
  upload_revid_location = private/.bzr-upload.revid
81
 
""")
82
 
 
83
 
 
84
 
# FIXME: Add more tests around invalid paths or relative paths that doesn't
85
 
# exist on remote (if only to get proper error messages) for
86
 
# 'upload_revid_location'
87
 
 
88
 
 
89
 
class BzrUploader(object):
90
 
 
91
 
    def __init__(self, branch, to_transport, outf, tree, rev_id,
92
 
                 quiet=False):
93
 
        self.branch = branch
94
 
        self.to_transport = to_transport
95
 
        self.outf = outf
96
 
        self.tree = tree
97
 
        self.rev_id = rev_id
98
 
        self.quiet = quiet
99
 
        self._pending_deletions = []
100
 
        self._pending_renames = []
101
 
        self._uploaded_revid = None
102
 
        self._ignored = None
103
 
 
104
 
    def _up_stat(self, relpath):
105
 
        return self.to_transport.stat(urlutils.escape(relpath))
106
 
 
107
 
    def _up_rename(self, old_path, new_path):
108
 
        return self.to_transport.rename(urlutils.escape(old_path),
109
 
                                        urlutils.escape(new_path))
110
 
 
111
 
    def _up_delete(self, relpath):
112
 
        return self.to_transport.delete(urlutils.escape(relpath))
113
 
 
114
 
    def _up_delete_tree(self, relpath):
115
 
        return self.to_transport.delete_tree(urlutils.escape(relpath))
116
 
 
117
 
    def _up_mkdir(self, relpath, mode):
118
 
        return self.to_transport.mkdir(urlutils.escape(relpath), mode)
119
 
 
120
 
    def _up_rmdir(self, relpath):
121
 
        return self.to_transport.rmdir(urlutils.escape(relpath))
122
 
 
123
 
    def _up_put_bytes(self, relpath, bytes, mode):
124
 
        self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
125
 
 
126
 
    def _up_get_bytes(self, relpath):
127
 
        return self.to_transport.get_bytes(urlutils.escape(relpath))
128
 
 
129
 
    def set_uploaded_revid(self, rev_id):
130
 
        # XXX: Add tests for concurrent updates, etc.
131
 
        revid_path = self.branch.get_config_stack().get('upload_revid_location')
132
 
        self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
133
 
        self._uploaded_revid = rev_id
134
 
 
135
 
    def get_uploaded_revid(self):
136
 
        if self._uploaded_revid is None:
137
 
            revid_path = self.branch.get_config_stack(
138
 
                ).get('upload_revid_location')
139
 
            try:
140
 
                self._uploaded_revid = self._up_get_bytes(revid_path)
141
 
            except errors.NoSuchFile:
142
 
                # We have not uploaded to here.
143
 
                self._uploaded_revid = revision.NULL_REVISION
144
 
        return self._uploaded_revid
145
 
 
146
 
    def _get_ignored(self):
147
 
        if self._ignored is None:
148
 
            try:
149
 
                ignore_file_path = '.bzrignore-upload'
150
 
                ignore_file = self.tree.get_file(ignore_file_path)
151
 
            except errors.NoSuchFile:
152
 
                ignored_patterns = []
153
 
            else:
154
 
                ignored_patterns = ignores.parse_ignore_file(ignore_file)
155
 
            self._ignored = globbing.Globster(ignored_patterns)
156
 
        return self._ignored
157
 
 
158
 
    def is_ignored(self, relpath):
159
 
        glob = self._get_ignored()
160
 
        ignored = glob.match(relpath)
161
 
        import os
162
 
        if not ignored:
163
 
            # We still need to check that all parents are not ignored
164
 
            dir = os.path.dirname(relpath)
165
 
            while dir and not ignored:
166
 
                ignored = glob.match(dir)
167
 
                if not ignored:
168
 
                    dir = os.path.dirname(dir)
169
 
        return ignored
170
 
 
171
 
    def upload_file(self, old_relpath, new_relpath, mode=None):
172
 
        if mode is None:
173
 
            if self.tree.is_executable(new_relpath):
174
 
                mode = 0o775
175
 
            else:
176
 
                mode = 0o664
177
 
        if not self.quiet:
178
 
            self.outf.write('Uploading %s\n' % old_relpath)
179
 
        self._up_put_bytes(
180
 
            old_relpath, self.tree.get_file_text(new_relpath), mode)
181
 
 
182
 
    def _force_clear(self, relpath):
183
 
        try:
184
 
            st = self._up_stat(relpath)
185
 
            if stat.S_ISDIR(st.st_mode):
186
 
                # A simple rmdir may not be enough
187
 
                if not self.quiet:
188
 
                    self.outf.write('Clearing %s/%s\n' % (
189
 
                        self.to_transport.external_url(), relpath))
190
 
                self._up_delete_tree(relpath)
191
 
            elif stat.S_ISLNK(st.st_mode):
192
 
                if not self.quiet:
193
 
                    self.outf.write('Clearing %s/%s\n' % (
194
 
                        self.to_transport.external_url(), relpath))
195
 
                self._up_delete(relpath)
196
 
        except errors.PathError:
197
 
            pass
198
 
 
199
 
    def upload_file_robustly(self, relpath, mode=None):
200
 
        """Upload a file, clearing the way on the remote side.
201
 
 
202
 
        When doing a full upload, it may happen that a directory exists where
203
 
        we want to put our file.
204
 
        """
205
 
        self._force_clear(relpath)
206
 
        self.upload_file(relpath, relpath, mode)
207
 
 
208
 
    def upload_symlink(self, relpath, target):
209
 
        self.to_transport.symlink(target, relpath)
210
 
 
211
 
    def upload_symlink_robustly(self, relpath, target):
212
 
        """Handle uploading symlinks.
213
 
        """
214
 
        self._force_clear(relpath)
215
 
        # Target might not be there at this time; dummy file should be
216
 
        # overwritten at some point, possibly by another upload.
217
 
        target = osutils.normpath(osutils.pathjoin(
218
 
            osutils.dirname(relpath),
219
 
            target)
220
 
        )
221
 
        self.upload_symlink(relpath, target)
222
 
 
223
 
    def make_remote_dir(self, relpath, mode=None):
224
 
        if mode is None:
225
 
            mode = 0o775
226
 
        self._up_mkdir(relpath, mode)
227
 
 
228
 
    def make_remote_dir_robustly(self, relpath, mode=None):
229
 
        """Create a remote directory, clearing the way on the remote side.
230
 
 
231
 
        When doing a full upload, it may happen that a file exists where we
232
 
        want to create our directory.
233
 
        """
234
 
        try:
235
 
            st = self._up_stat(relpath)
236
 
            if not stat.S_ISDIR(st.st_mode):
237
 
                if not self.quiet:
238
 
                    self.outf.write('Deleting %s/%s\n' % (
239
 
                        self.to_transport.external_url(), relpath))
240
 
                self._up_delete(relpath)
241
 
            else:
242
 
                # Ok the remote dir already exists, nothing to do
243
 
                return
244
 
        except errors.PathError:
245
 
            pass
246
 
        self.make_remote_dir(relpath, mode)
247
 
 
248
 
    def delete_remote_file(self, relpath):
249
 
        if not self.quiet:
250
 
            self.outf.write('Deleting %s\n' % relpath)
251
 
        self._up_delete(relpath)
252
 
 
253
 
    def delete_remote_dir(self, relpath):
254
 
        if not self.quiet:
255
 
            self.outf.write('Deleting %s\n' % relpath)
256
 
        self._up_rmdir(relpath)
257
 
        # XXX: Add a test where a subdir is ignored but we still want to
258
 
        # delete the dir -- vila 100106
259
 
 
260
 
    def delete_remote_dir_maybe(self, relpath):
261
 
        """Try to delete relpath, keeping failures to retry later."""
262
 
        try:
263
 
            self._up_rmdir(relpath)
264
 
        # any kind of PathError would be OK, though we normally expect
265
 
        # DirectoryNotEmpty
266
 
        except errors.PathError:
267
 
            self._pending_deletions.append(relpath)
268
 
 
269
 
    def finish_deletions(self):
270
 
        if self._pending_deletions:
271
 
            # Process the previously failed deletions in reverse order to
272
 
            # delete children before parents
273
 
            for relpath in reversed(self._pending_deletions):
274
 
                self._up_rmdir(relpath)
275
 
            # The following shouldn't be needed since we use it once per
276
 
            # upload, but better safe than sorry ;-)
277
 
            self._pending_deletions = []
278
 
 
279
 
    def rename_remote(self, old_relpath, new_relpath):
280
 
        """Rename a remote file or directory taking care of collisions.
281
 
 
282
 
        To avoid collisions during bulk renames, each renamed target is
283
 
        temporarily assigned a unique name. When all renames have been done,
284
 
        each target get its proper name.
285
 
        """
286
 
        # We generate a sufficiently random name to *assume* that
287
 
        # no collisions will occur and don't worry about it (nor
288
 
        # handle it).
289
 
        import os
290
 
        import random
291
 
        import time
292
 
 
293
 
        stamp = '.tmp.%.9f.%d.%d' % (time.time(),
294
 
                                     os.getpid(),
295
 
                                     random.randint(0, 0x7FFFFFFF))
296
 
        if not self.quiet:
297
 
            self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
298
 
        self._up_rename(old_relpath, stamp)
299
 
        self._pending_renames.append((stamp, new_relpath))
300
 
 
301
 
    def finish_renames(self):
302
 
        for (stamp, new_path) in self._pending_renames:
303
 
            self._up_rename(stamp, new_path)
304
 
        # The following shouldn't be needed since we use it once per upload,
305
 
        # but better safe than sorry ;-)
306
 
        self._pending_renames = []
307
 
 
308
 
    def upload_full_tree(self):
309
 
        self.to_transport.ensure_base()  # XXX: Handle errors (add
310
 
        # --create-prefix option ?)
311
 
        with self.tree.lock_read():
312
 
            for relpath, ie in self.tree.iter_entries_by_dir():
313
 
                if relpath in ('', '.bzrignore', '.bzrignore-upload'):
314
 
                    # skip root ('')
315
 
                    # .bzrignore and .bzrignore-upload have no meaning outside
316
 
                    # a working tree so do not upload them
317
 
                    continue
318
 
                if self.is_ignored(relpath):
319
 
                    if not self.quiet:
320
 
                        self.outf.write('Ignoring %s\n' % relpath)
321
 
                    continue
322
 
                if ie.kind == 'file':
323
 
                    self.upload_file_robustly(relpath)
324
 
                elif ie.kind == 'symlink':
325
 
                    try:
326
 
                        self.upload_symlink_robustly(
327
 
                            relpath, ie.symlink_target)
328
 
                    except errors.TransportNotPossible:
329
 
                        if not self.quiet:
330
 
                            target = self.tree.path_content_summary(relpath)[3]
331
 
                            self.outf.write('Not uploading symlink %s -> %s\n'
332
 
                                            % (relpath, target))
333
 
                elif ie.kind == 'directory':
334
 
                    self.make_remote_dir_robustly(relpath)
335
 
                else:
336
 
                    raise NotImplementedError
337
 
            self.set_uploaded_revid(self.rev_id)
338
 
 
339
 
    def upload_tree(self):
340
 
        # If we can't find the revid file on the remote location, upload the
341
 
        # full tree instead
342
 
        rev_id = self.get_uploaded_revid()
343
 
 
344
 
        if rev_id == revision.NULL_REVISION:
345
 
            if not self.quiet:
346
 
                self.outf.write('No uploaded revision id found,'
347
 
                                ' switching to full upload\n')
348
 
            self.upload_full_tree()
349
 
            # We're done
350
 
            return
351
 
 
352
 
        # Check if the revision hasn't already been uploaded
353
 
        if rev_id == self.rev_id:
354
 
            if not self.quiet:
355
 
                self.outf.write('Remote location already up to date\n')
356
 
 
357
 
        from_tree = self.branch.repository.revision_tree(rev_id)
358
 
        self.to_transport.ensure_base()  # XXX: Handle errors (add
359
 
        # --create-prefix option ?)
360
 
        changes = self.tree.changes_from(from_tree)
361
 
        with self.tree.lock_read():
362
 
            for change in changes.removed:
363
 
                if self.is_ignored(change.path[0]):
364
 
                    if not self.quiet:
365
 
                        self.outf.write('Ignoring %s\n' % change.path[0])
366
 
                    continue
367
 
                if change.kind[0] == 'file':
368
 
                    self.delete_remote_file(change.path[0])
369
 
                elif change.kind[0] == 'directory':
370
 
                    self.delete_remote_dir_maybe(change.path[0])
371
 
                elif change.kind[0] == 'symlink':
372
 
                    self.delete_remote_file(change.path[0])
373
 
                else:
374
 
                    raise NotImplementedError
375
 
 
376
 
            for change in changes.renamed:
377
 
                if self.is_ignored(change.path[0]) and self.is_ignored(change.path[1]):
378
 
                    if not self.quiet:
379
 
                        self.outf.write('Ignoring %s\n' % change.path[0])
380
 
                        self.outf.write('Ignoring %s\n' % change.path[1])
381
 
                    continue
382
 
                if change.changed_content:
383
 
                    # We update the change.path[0] content because renames and
384
 
                    # deletions are differed.
385
 
                    self.upload_file(change.path[0], change.path[1])
386
 
                self.rename_remote(change.path[0], change.path[1])
387
 
            self.finish_renames()
388
 
            self.finish_deletions()
389
 
 
390
 
            for change in changes.kind_changed:
391
 
                if self.is_ignored(change.path[1]):
392
 
                    if not self.quiet:
393
 
                        self.outf.write('Ignoring %s\n' % change.path[1])
394
 
                    continue
395
 
                if change.kind[0] in ('file', 'symlink'):
396
 
                    self.delete_remote_file(change.path[0])
397
 
                elif change.kind[0] == 'directory':
398
 
                    self.delete_remote_dir(change.path[0])
399
 
                else:
400
 
                    raise NotImplementedError
401
 
 
402
 
                if change.kind[1] == 'file':
403
 
                    self.upload_file(change.path[1], change.path[1])
404
 
                elif change.kind[1] == 'symlink':
405
 
                    target = self.tree.get_symlink_target(change.path[1])
406
 
                    self.upload_symlink(change.path[1], target)
407
 
                elif change.kind[1] == 'directory':
408
 
                    self.make_remote_dir(change.path[1])
409
 
                else:
410
 
                    raise NotImplementedError
411
 
 
412
 
            for change in changes.added + changes.copied:
413
 
                if self.is_ignored(change.path[1]):
414
 
                    if not self.quiet:
415
 
                        self.outf.write('Ignoring %s\n' % change.path[1])
416
 
                    continue
417
 
                if change.kind[1] == 'file':
418
 
                    self.upload_file(change.path[1], change.path[1])
419
 
                elif change.kind[1] == 'directory':
420
 
                    self.make_remote_dir(change.path[1])
421
 
                elif change.kind[1] == 'symlink':
422
 
                    target = self.tree.get_symlink_target(change.path[1])
423
 
                    try:
424
 
                        self.upload_symlink(change.path[1], target)
425
 
                    except errors.TransportNotPossible:
426
 
                        if not self.quiet:
427
 
                            self.outf.write('Not uploading symlink %s -> %s\n'
428
 
                                            % (change.path[1], target))
429
 
                else:
430
 
                    raise NotImplementedError
431
 
 
432
 
            # XXX: Add a test for exec_change
433
 
            for change in changes.modified:
434
 
                if self.is_ignored(change.path[1]):
435
 
                    if not self.quiet:
436
 
                        self.outf.write('Ignoring %s\n' % change.path[1])
437
 
                    continue
438
 
                if change.kind[1] == 'file':
439
 
                    self.upload_file(change.path[1], change.path[1])
440
 
                elif change.kind[1] == 'symlink':
441
 
                    target = self.tree.get_symlink_target(change.path[1])
442
 
                    self.upload_symlink(change.path[1], target)
443
 
                else:
444
 
                    raise NotImplementedError
445
 
 
446
 
            self.set_uploaded_revid(self.rev_id)
447
 
 
448
 
 
449
 
class CannotUploadToWorkingTree(errors.BzrCommandError):
450
 
 
451
 
    _fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
452
 
 
453
 
 
454
 
class DivergedUploadedTree(errors.BzrCommandError):
455
 
 
456
 
    _fmt = ("Your branch (%(revid)s)"
457
 
            " and the uploaded tree (%(uploaded_revid)s) have diverged: ")
458
 
 
459
 
 
460
 
class cmd_upload(commands.Command):
461
 
    """Upload a working tree, as a whole or incrementally.
462
 
 
463
 
    If no destination is specified use the last one used.
464
 
    If no revision is specified upload the changes since the last upload.
465
 
 
466
 
    Changes include files added, renamed, modified or removed.
467
 
    """
468
 
    _see_also = ['plugins/upload']
469
 
    takes_args = ['location?']
470
 
    takes_options = [
471
 
        'revision',
472
 
        'remember',
473
 
        'overwrite',
474
 
        option.Option('full', 'Upload the full working tree.'),
475
 
        option.Option('quiet', 'Do not output what is being done.',
476
 
                      short_name='q'),
477
 
        option.Option('directory',
478
 
                      help='Branch to upload from, '
479
 
                      'rather than the one containing the working directory.',
480
 
                      short_name='d',
481
 
                      type=str,
482
 
                      ),
483
 
        option.Option('auto',
484
 
                      'Trigger an upload from this branch whenever the tip '
485
 
                      'revision changes.')
486
 
        ]
487
 
 
488
 
    def run(self, location=None, full=False, revision=None, remember=None,
489
 
            directory=None, quiet=False, auto=None, overwrite=False
490
 
            ):
491
 
        if directory is None:
492
 
            directory = u'.'
493
 
 
494
 
        (wt, branch,
495
 
         relpath) = controldir.ControlDir.open_containing_tree_or_branch(
496
 
             directory)
497
 
 
498
 
        if wt:
499
 
            locked = wt
500
 
        else:
501
 
            locked = branch
502
 
        with locked.lock_read():
503
 
            if wt:
504
 
                changes = wt.changes_from(wt.basis_tree())
505
 
 
506
 
                if revision is None and changes.has_changed():
507
 
                    raise errors.UncommittedChanges(wt)
508
 
 
509
 
            conf = branch.get_config_stack()
510
 
            if location is None:
511
 
                stored_loc = conf.get('upload_location')
512
 
                if stored_loc is None:
513
 
                    raise errors.BzrCommandError(
514
 
                        'No upload location known or specified.')
515
 
                else:
516
 
                    # FIXME: Not currently tested
517
 
                    display_url = urlutils.unescape_for_display(stored_loc,
518
 
                                                                self.outf.encoding)
519
 
                    self.outf.write("Using saved location: %s\n" % display_url)
520
 
                    location = stored_loc
521
 
 
522
 
            to_transport = transport.get_transport(location)
523
 
 
524
 
            # Check that we are not uploading to a existing working tree.
525
 
            try:
526
 
                to_bzr_dir = controldir.ControlDir.open_from_transport(
527
 
                    to_transport)
528
 
                has_wt = to_bzr_dir.has_workingtree()
529
 
            except errors.NotBranchError:
530
 
                has_wt = False
531
 
            except errors.NotLocalUrl:
532
 
                # The exception raised is a bit weird... but that's life.
533
 
                has_wt = True
534
 
 
535
 
            if has_wt:
536
 
                raise CannotUploadToWorkingTree(url=location)
537
 
            if revision is None:
538
 
                rev_id = branch.last_revision()
539
 
            else:
540
 
                if len(revision) != 1:
541
 
                    raise errors.BzrCommandError(
542
 
                        'bzr upload --revision takes exactly 1 argument')
543
 
                rev_id = revision[0].in_history(branch).rev_id
544
 
 
545
 
            tree = branch.repository.revision_tree(rev_id)
546
 
 
547
 
            uploader = BzrUploader(branch, to_transport, self.outf, tree,
548
 
                                   rev_id, quiet=quiet)
549
 
 
550
 
            if not overwrite:
551
 
                prev_uploaded_rev_id = uploader.get_uploaded_revid()
552
 
                graph = branch.repository.get_graph()
553
 
                if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
554
 
                    raise DivergedUploadedTree(
555
 
                        revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
556
 
            if full:
557
 
                uploader.upload_full_tree()
558
 
            else:
559
 
                uploader.upload_tree()
560
 
 
561
 
        # We uploaded successfully, remember it
562
 
        with branch.lock_write():
563
 
            upload_location = conf.get('upload_location')
564
 
            if upload_location is None or remember:
565
 
                conf.set('upload_location',
566
 
                         urlutils.unescape(to_transport.base))
567
 
            if auto is not None:
568
 
                conf.set('upload_auto', auto)