/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/plugins/upload/cmds.py

  • Committer: Jelmer Vernooij
  • Date: 2020-04-05 19:11:34 UTC
  • mto: (7490.7.16 work)
  • mto: This revision was merged to the branch mainline in revision 7501.
  • Revision ID: jelmer@jelmer.uk-20200405191134-0aebh8ikiwygxma5
Populate the .gitignore file.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/plugins/upload/cmds.py
 
1
# Copyright (C) 2011, 2012 Canonical Ltd
 
2
#
 
3
# This program is free software; you can redistribute it and/or modify
 
4
# it under the terms of the GNU General Public License as published by
 
5
# the Free Software Foundation; either version 2 of the License, or
 
6
# (at your option) any later version.
 
7
#
 
8
# This program is distributed in the hope that it will be useful,
 
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
11
# GNU General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
16
 
 
17
"""bzr-upload command implementations."""
 
18
 
 
19
from __future__ import absolute_import
 
20
 
 
21
from ... import (
 
22
    commands,
 
23
    config,
 
24
    errors,
 
25
    lazy_import,
 
26
    option,
 
27
    osutils,
 
28
    )
 
29
lazy_import.lazy_import(globals(), """
 
30
import stat
 
31
 
 
32
from breezy import (
 
33
    controldir,
 
34
    globbing,
 
35
    ignores,
 
36
    revision,
 
37
    transport,
 
38
    urlutils,
 
39
    )
 
40
""")
 
41
 
 
42
from ...sixish import (
 
43
    text_type,
 
44
    )
 
45
 
 
46
auto_option = config.Option(
 
47
    'upload_auto', default=False, from_unicode=config.bool_from_store,
 
48
    help="""\
 
49
Whether upload should occur when the tip of the branch changes.
 
50
""")
 
51
auto_quiet_option = config.Option(
 
52
    'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
 
53
    help="""\
 
54
Whether upload should occur quietly.
 
55
""")
 
56
location_option = config.Option(
 
57
    'upload_location', default=None,
 
58
    help="""\
 
59
The url to upload the working tree to.
 
60
""")
 
61
revid_location_option = config.Option(
 
62
    'upload_revid_location', default=u'.bzr-upload.revid',
 
63
    help="""\
 
64
The relative path to be used to store the uploaded revid.
 
65
 
 
66
The only bzr-related info uploaded with the working tree is the corresponding
 
67
revision id. The uploaded working tree is not linked to any other bzr data.
 
68
 
 
69
If the layout of your remote server is such that you can't write in the
 
70
root directory but only in the directories inside that root, you will need
 
71
to use the 'upload_revid_location' configuration variable to specify the
 
72
relative path to be used. That configuration variable can be specified in
 
73
locations.conf or branch.conf.
 
74
 
 
75
For example, given the following layout:
 
76
 
 
77
  Project/
 
78
    private/
 
79
    public/
 
80
 
 
81
you may have write access in 'private' and 'public' but in 'Project'
 
82
itself. In that case, you can add the following in your locations.conf or
 
83
branch.conf file:
 
84
 
 
85
  upload_revid_location = private/.bzr-upload.revid
 
86
""")
 
87
 
 
88
 
 
89
# FIXME: Add more tests around invalid paths or relative paths that doesn't
 
90
# exist on remote (if only to get proper error messages) for
 
91
# 'upload_revid_location'
 
92
 
 
93
 
 
94
class BzrUploader(object):
 
95
 
 
96
    def __init__(self, branch, to_transport, outf, tree, rev_id,
 
97
                 quiet=False):
 
98
        self.branch = branch
 
99
        self.to_transport = to_transport
 
100
        self.outf = outf
 
101
        self.tree = tree
 
102
        self.rev_id = rev_id
 
103
        self.quiet = quiet
 
104
        self._pending_deletions = []
 
105
        self._pending_renames = []
 
106
        self._uploaded_revid = None
 
107
        self._ignored = None
 
108
 
 
109
    def _up_stat(self, relpath):
 
110
        return self.to_transport.stat(urlutils.escape(relpath))
 
111
 
 
112
    def _up_rename(self, old_path, new_path):
 
113
        return self.to_transport.rename(urlutils.escape(old_path),
 
114
                                        urlutils.escape(new_path))
 
115
 
 
116
    def _up_delete(self, relpath):
 
117
        return self.to_transport.delete(urlutils.escape(relpath))
 
118
 
 
119
    def _up_delete_tree(self, relpath):
 
120
        return self.to_transport.delete_tree(urlutils.escape(relpath))
 
121
 
 
122
    def _up_mkdir(self, relpath, mode):
 
123
        return self.to_transport.mkdir(urlutils.escape(relpath), mode)
 
124
 
 
125
    def _up_rmdir(self, relpath):
 
126
        return self.to_transport.rmdir(urlutils.escape(relpath))
 
127
 
 
128
    def _up_put_bytes(self, relpath, bytes, mode):
 
129
        self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
 
130
 
 
131
    def _up_get_bytes(self, relpath):
 
132
        return self.to_transport.get_bytes(urlutils.escape(relpath))
 
133
 
 
134
    def set_uploaded_revid(self, rev_id):
 
135
        # XXX: Add tests for concurrent updates, etc.
 
136
        revid_path = self.branch.get_config_stack().get('upload_revid_location')
 
137
        self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
 
138
        self._uploaded_revid = rev_id
 
139
 
 
140
    def get_uploaded_revid(self):
 
141
        if self._uploaded_revid is None:
 
142
            revid_path = self.branch.get_config_stack(
 
143
                ).get('upload_revid_location')
 
144
            try:
 
145
                self._uploaded_revid = self._up_get_bytes(revid_path)
 
146
            except errors.NoSuchFile:
 
147
                # We have not uploaded to here.
 
148
                self._uploaded_revid = revision.NULL_REVISION
 
149
        return self._uploaded_revid
 
150
 
 
151
    def _get_ignored(self):
 
152
        if self._ignored is None:
 
153
            try:
 
154
                ignore_file_path = '.bzrignore-upload'
 
155
                ignore_file = self.tree.get_file(ignore_file_path)
 
156
            except errors.NoSuchFile:
 
157
                ignored_patterns = []
 
158
            else:
 
159
                ignored_patterns = ignores.parse_ignore_file(ignore_file)
 
160
            self._ignored = globbing.Globster(ignored_patterns)
 
161
        return self._ignored
 
162
 
 
163
    def is_ignored(self, relpath):
 
164
        glob = self._get_ignored()
 
165
        ignored = glob.match(relpath)
 
166
        import os
 
167
        if not ignored:
 
168
            # We still need to check that all parents are not ignored
 
169
            dir = os.path.dirname(relpath)
 
170
            while dir and not ignored:
 
171
                ignored = glob.match(dir)
 
172
                if not ignored:
 
173
                    dir = os.path.dirname(dir)
 
174
        return ignored
 
175
 
 
176
    def upload_file(self, old_relpath, new_relpath, mode=None):
 
177
        if mode is None:
 
178
            if self.tree.is_executable(new_relpath):
 
179
                mode = 0o775
 
180
            else:
 
181
                mode = 0o664
 
182
        if not self.quiet:
 
183
            self.outf.write('Uploading %s\n' % old_relpath)
 
184
        self._up_put_bytes(
 
185
            old_relpath, self.tree.get_file_text(new_relpath), mode)
 
186
 
 
187
    def _force_clear(self, relpath):
 
188
        try:
 
189
            st = self._up_stat(relpath)
 
190
            if stat.S_ISDIR(st.st_mode):
 
191
                # A simple rmdir may not be enough
 
192
                if not self.quiet:
 
193
                    self.outf.write('Clearing %s/%s\n' % (
 
194
                        self.to_transport.external_url(), relpath))
 
195
                self._up_delete_tree(relpath)
 
196
            elif stat.S_ISLNK(st.st_mode):
 
197
                if not self.quiet:
 
198
                    self.outf.write('Clearing %s/%s\n' % (
 
199
                        self.to_transport.external_url(), relpath))
 
200
                self._up_delete(relpath)
 
201
        except errors.PathError:
 
202
            pass
 
203
 
 
204
    def upload_file_robustly(self, relpath, mode=None):
 
205
        """Upload a file, clearing the way on the remote side.
 
206
 
 
207
        When doing a full upload, it may happen that a directory exists where
 
208
        we want to put our file.
 
209
        """
 
210
        self._force_clear(relpath)
 
211
        self.upload_file(relpath, relpath, mode)
 
212
 
 
213
    def upload_symlink(self, relpath, target):
 
214
        self.to_transport.symlink(target, relpath)
 
215
 
 
216
    def upload_symlink_robustly(self, relpath, target):
 
217
        """Handle uploading symlinks.
 
218
        """
 
219
        self._force_clear(relpath)
 
220
        # Target might not be there at this time; dummy file should be
 
221
        # overwritten at some point, possibly by another upload.
 
222
        target = osutils.normpath(osutils.pathjoin(
 
223
            osutils.dirname(relpath),
 
224
            target)
 
225
        )
 
226
        self.upload_symlink(relpath, target)
 
227
 
 
228
    def make_remote_dir(self, relpath, mode=None):
 
229
        if mode is None:
 
230
            mode = 0o775
 
231
        self._up_mkdir(relpath, mode)
 
232
 
 
233
    def make_remote_dir_robustly(self, relpath, mode=None):
 
234
        """Create a remote directory, clearing the way on the remote side.
 
235
 
 
236
        When doing a full upload, it may happen that a file exists where we
 
237
        want to create our directory.
 
238
        """
 
239
        try:
 
240
            st = self._up_stat(relpath)
 
241
            if not stat.S_ISDIR(st.st_mode):
 
242
                if not self.quiet:
 
243
                    self.outf.write('Deleting %s/%s\n' % (
 
244
                        self.to_transport.external_url(), relpath))
 
245
                self._up_delete(relpath)
 
246
            else:
 
247
                # Ok the remote dir already exists, nothing to do
 
248
                return
 
249
        except errors.PathError:
 
250
            pass
 
251
        self.make_remote_dir(relpath, mode)
 
252
 
 
253
    def delete_remote_file(self, relpath):
 
254
        if not self.quiet:
 
255
            self.outf.write('Deleting %s\n' % relpath)
 
256
        self._up_delete(relpath)
 
257
 
 
258
    def delete_remote_dir(self, relpath):
 
259
        if not self.quiet:
 
260
            self.outf.write('Deleting %s\n' % relpath)
 
261
        self._up_rmdir(relpath)
 
262
        # XXX: Add a test where a subdir is ignored but we still want to
 
263
        # delete the dir -- vila 100106
 
264
 
 
265
    def delete_remote_dir_maybe(self, relpath):
 
266
        """Try to delete relpath, keeping failures to retry later."""
 
267
        try:
 
268
            self._up_rmdir(relpath)
 
269
        # any kind of PathError would be OK, though we normally expect
 
270
        # DirectoryNotEmpty
 
271
        except errors.PathError:
 
272
            self._pending_deletions.append(relpath)
 
273
 
 
274
    def finish_deletions(self):
 
275
        if self._pending_deletions:
 
276
            # Process the previously failed deletions in reverse order to
 
277
            # delete children before parents
 
278
            for relpath in reversed(self._pending_deletions):
 
279
                self._up_rmdir(relpath)
 
280
            # The following shouldn't be needed since we use it once per
 
281
            # upload, but better safe than sorry ;-)
 
282
            self._pending_deletions = []
 
283
 
 
284
    def rename_remote(self, old_relpath, new_relpath):
 
285
        """Rename a remote file or directory taking care of collisions.
 
286
 
 
287
        To avoid collisions during bulk renames, each renamed target is
 
288
        temporarily assigned a unique name. When all renames have been done,
 
289
        each target get its proper name.
 
290
        """
 
291
        # We generate a sufficiently random name to *assume* that
 
292
        # no collisions will occur and don't worry about it (nor
 
293
        # handle it).
 
294
        import os
 
295
        import random
 
296
        import time
 
297
 
 
298
        stamp = '.tmp.%.9f.%d.%d' % (time.time(),
 
299
                                     os.getpid(),
 
300
                                     random.randint(0, 0x7FFFFFFF))
 
301
        if not self.quiet:
 
302
            self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
 
303
        self._up_rename(old_relpath, stamp)
 
304
        self._pending_renames.append((stamp, new_relpath))
 
305
 
 
306
    def finish_renames(self):
 
307
        for (stamp, new_path) in self._pending_renames:
 
308
            self._up_rename(stamp, new_path)
 
309
        # The following shouldn't be needed since we use it once per upload,
 
310
        # but better safe than sorry ;-)
 
311
        self._pending_renames = []
 
312
 
 
313
    def upload_full_tree(self):
 
314
        self.to_transport.ensure_base()  # XXX: Handle errors (add
 
315
        # --create-prefix option ?)
 
316
        with self.tree.lock_read():
 
317
            for relpath, ie in self.tree.iter_entries_by_dir():
 
318
                if relpath in ('', '.bzrignore', '.bzrignore-upload'):
 
319
                    # skip root ('')
 
320
                    # .bzrignore and .bzrignore-upload have no meaning outside
 
321
                    # a working tree so do not upload them
 
322
                    continue
 
323
                if self.is_ignored(relpath):
 
324
                    if not self.quiet:
 
325
                        self.outf.write('Ignoring %s\n' % relpath)
 
326
                    continue
 
327
                if ie.kind == 'file':
 
328
                    self.upload_file_robustly(relpath)
 
329
                elif ie.kind == 'symlink':
 
330
                    try:
 
331
                        self.upload_symlink_robustly(
 
332
                            relpath, ie.symlink_target)
 
333
                    except errors.TransportNotPossible:
 
334
                        if not self.quiet:
 
335
                            target = self.tree.path_content_summary(relpath)[3]
 
336
                            self.outf.write('Not uploading symlink %s -> %s\n'
 
337
                                            % (relpath, target))
 
338
                elif ie.kind == 'directory':
 
339
                    self.make_remote_dir_robustly(relpath)
 
340
                else:
 
341
                    raise NotImplementedError
 
342
            self.set_uploaded_revid(self.rev_id)
 
343
 
 
344
    def upload_tree(self):
 
345
        # If we can't find the revid file on the remote location, upload the
 
346
        # full tree instead
 
347
        rev_id = self.get_uploaded_revid()
 
348
 
 
349
        if rev_id == revision.NULL_REVISION:
 
350
            if not self.quiet:
 
351
                self.outf.write('No uploaded revision id found,'
 
352
                                ' switching to full upload\n')
 
353
            self.upload_full_tree()
 
354
            # We're done
 
355
            return
 
356
 
 
357
        # Check if the revision hasn't already been uploaded
 
358
        if rev_id == self.rev_id:
 
359
            if not self.quiet:
 
360
                self.outf.write('Remote location already up to date\n')
 
361
 
 
362
        from_tree = self.branch.repository.revision_tree(rev_id)
 
363
        self.to_transport.ensure_base()  # XXX: Handle errors (add
 
364
        # --create-prefix option ?)
 
365
        changes = self.tree.changes_from(from_tree)
 
366
        with self.tree.lock_read():
 
367
            for change in changes.removed:
 
368
                if self.is_ignored(change.path[0]):
 
369
                    if not self.quiet:
 
370
                        self.outf.write('Ignoring %s\n' % change.path[0])
 
371
                    continue
 
372
                if change.kind[0] == 'file':
 
373
                    self.delete_remote_file(change.path[0])
 
374
                elif change.kind[0] == 'directory':
 
375
                    self.delete_remote_dir_maybe(change.path[0])
 
376
                elif change.kind[0] == 'symlink':
 
377
                    self.delete_remote_file(change.path[0])
 
378
                else:
 
379
                    raise NotImplementedError
 
380
 
 
381
            for change in changes.renamed:
 
382
                if self.is_ignored(change.path[0]) and self.is_ignored(change.path[1]):
 
383
                    if not self.quiet:
 
384
                        self.outf.write('Ignoring %s\n' % change.path[0])
 
385
                        self.outf.write('Ignoring %s\n' % change.path[1])
 
386
                    continue
 
387
                if change.changed_content:
 
388
                    # We update the change.path[0] content because renames and
 
389
                    # deletions are differed.
 
390
                    self.upload_file(change.path[0], change.path[1])
 
391
                self.rename_remote(change.path[0], change.path[1])
 
392
            self.finish_renames()
 
393
            self.finish_deletions()
 
394
 
 
395
            for change in changes.kind_changed:
 
396
                if self.is_ignored(change.path[1]):
 
397
                    if not self.quiet:
 
398
                        self.outf.write('Ignoring %s\n' % change.path[1])
 
399
                    continue
 
400
                if change.kind[0] in ('file', 'symlink'):
 
401
                    self.delete_remote_file(change.path[0])
 
402
                elif change.kind[0] == 'directory':
 
403
                    self.delete_remote_dir(change.path[0])
 
404
                else:
 
405
                    raise NotImplementedError
 
406
 
 
407
                if change.kind[1] == 'file':
 
408
                    self.upload_file(change.path[1], change.path[1])
 
409
                elif change.kind[1] == 'symlink':
 
410
                    target = self.tree.get_symlink_target(change.path[1])
 
411
                    self.upload_symlink(change.path[1], target)
 
412
                elif change.kind[1] == 'directory':
 
413
                    self.make_remote_dir(change.path[1])
 
414
                else:
 
415
                    raise NotImplementedError
 
416
 
 
417
            for change in changes.added + changes.copied:
 
418
                if self.is_ignored(change.path[1]):
 
419
                    if not self.quiet:
 
420
                        self.outf.write('Ignoring %s\n' % change.path[1])
 
421
                    continue
 
422
                if change.kind[1] == 'file':
 
423
                    self.upload_file(change.path[1], change.path[1])
 
424
                elif change.kind[1] == 'directory':
 
425
                    self.make_remote_dir(change.path[1])
 
426
                elif change.kind[1] == 'symlink':
 
427
                    target = self.tree.get_symlink_target(change.path[1])
 
428
                    try:
 
429
                        self.upload_symlink(change.path[1], target)
 
430
                    except errors.TransportNotPossible:
 
431
                        if not self.quiet:
 
432
                            self.outf.write('Not uploading symlink %s -> %s\n'
 
433
                                            % (change.path[1], target))
 
434
                else:
 
435
                    raise NotImplementedError
 
436
 
 
437
            # XXX: Add a test for exec_change
 
438
            for change in changes.modified:
 
439
                if self.is_ignored(change.path[1]):
 
440
                    if not self.quiet:
 
441
                        self.outf.write('Ignoring %s\n' % change.path[1])
 
442
                    continue
 
443
                if change.kind[1] == 'file':
 
444
                    self.upload_file(change.path[1], change.path[1])
 
445
                elif change.kind[1] == 'symlink':
 
446
                    target = self.tree.get_symlink_target(change.path[1])
 
447
                    self.upload_symlink(change.path[1], target)
 
448
                else:
 
449
                    raise NotImplementedError
 
450
 
 
451
            self.set_uploaded_revid(self.rev_id)
 
452
 
 
453
 
 
454
class CannotUploadToWorkingTree(errors.BzrCommandError):
 
455
 
 
456
    _fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
 
457
 
 
458
 
 
459
class DivergedUploadedTree(errors.BzrCommandError):
 
460
 
 
461
    _fmt = ("Your branch (%(revid)s)"
 
462
            " and the uploaded tree (%(uploaded_revid)s) have diverged: ")
 
463
 
 
464
 
 
465
class cmd_upload(commands.Command):
 
466
    """Upload a working tree, as a whole or incrementally.
 
467
 
 
468
    If no destination is specified use the last one used.
 
469
    If no revision is specified upload the changes since the last upload.
 
470
 
 
471
    Changes include files added, renamed, modified or removed.
 
472
    """
 
473
    _see_also = ['plugins/upload']
 
474
    takes_args = ['location?']
 
475
    takes_options = [
 
476
        'revision',
 
477
        'remember',
 
478
        'overwrite',
 
479
        option.Option('full', 'Upload the full working tree.'),
 
480
        option.Option('quiet', 'Do not output what is being done.',
 
481
                      short_name='q'),
 
482
        option.Option('directory',
 
483
                      help='Branch to upload from, '
 
484
                      'rather than the one containing the working directory.',
 
485
                      short_name='d',
 
486
                      type=text_type,
 
487
                      ),
 
488
        option.Option('auto',
 
489
                      'Trigger an upload from this branch whenever the tip '
 
490
                      'revision changes.')
 
491
        ]
 
492
 
 
493
    def run(self, location=None, full=False, revision=None, remember=None,
 
494
            directory=None, quiet=False, auto=None, overwrite=False
 
495
            ):
 
496
        if directory is None:
 
497
            directory = u'.'
 
498
 
 
499
        (wt, branch,
 
500
         relpath) = controldir.ControlDir.open_containing_tree_or_branch(
 
501
             directory)
 
502
 
 
503
        if wt:
 
504
            locked = wt
 
505
        else:
 
506
            locked = branch
 
507
        with locked.lock_read():
 
508
            if wt:
 
509
                changes = wt.changes_from(wt.basis_tree())
 
510
 
 
511
                if revision is None and changes.has_changed():
 
512
                    raise errors.UncommittedChanges(wt)
 
513
 
 
514
            conf = branch.get_config_stack()
 
515
            if location is None:
 
516
                stored_loc = conf.get('upload_location')
 
517
                if stored_loc is None:
 
518
                    raise errors.BzrCommandError(
 
519
                        'No upload location known or specified.')
 
520
                else:
 
521
                    # FIXME: Not currently tested
 
522
                    display_url = urlutils.unescape_for_display(stored_loc,
 
523
                                                                self.outf.encoding)
 
524
                    self.outf.write("Using saved location: %s\n" % display_url)
 
525
                    location = stored_loc
 
526
 
 
527
            to_transport = transport.get_transport(location)
 
528
 
 
529
            # Check that we are not uploading to a existing working tree.
 
530
            try:
 
531
                to_bzr_dir = controldir.ControlDir.open_from_transport(
 
532
                    to_transport)
 
533
                has_wt = to_bzr_dir.has_workingtree()
 
534
            except errors.NotBranchError:
 
535
                has_wt = False
 
536
            except errors.NotLocalUrl:
 
537
                # The exception raised is a bit weird... but that's life.
 
538
                has_wt = True
 
539
 
 
540
            if has_wt:
 
541
                raise CannotUploadToWorkingTree(url=location)
 
542
            if revision is None:
 
543
                rev_id = branch.last_revision()
 
544
            else:
 
545
                if len(revision) != 1:
 
546
                    raise errors.BzrCommandError(
 
547
                        'bzr upload --revision takes exactly 1 argument')
 
548
                rev_id = revision[0].in_history(branch).rev_id
 
549
 
 
550
            tree = branch.repository.revision_tree(rev_id)
 
551
 
 
552
            uploader = BzrUploader(branch, to_transport, self.outf, tree,
 
553
                                   rev_id, quiet=quiet)
 
554
 
 
555
            if not overwrite:
 
556
                prev_uploaded_rev_id = uploader.get_uploaded_revid()
 
557
                graph = branch.repository.get_graph()
 
558
                if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
 
559
                    raise DivergedUploadedTree(
 
560
                        revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
 
561
            if full:
 
562
                uploader.upload_full_tree()
 
563
            else:
 
564
                uploader.upload_tree()
 
565
 
 
566
        # We uploaded successfully, remember it
 
567
        with branch.lock_write():
 
568
            upload_location = conf.get('upload_location')
 
569
            if upload_location is None or remember:
 
570
                conf.set('upload_location',
 
571
                         urlutils.unescape(to_transport.base))
 
572
            if auto is not None:
 
573
                conf.set('upload_auto', auto)