/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/plugins/upload/cmds.py

  • Committer: Richard Wilbur
  • Date: 2016-02-04 19:07:28 UTC
  • mto: This revision was merged to the branch mainline in revision 6618.
  • Revision ID: richard.wilbur@gmail.com-20160204190728-p0zvfii6zase0fw7
Update COPYING.txt from the original http://www.gnu.org/licenses/gpl-2.0.txt  (Only differences were in whitespace.)  Thanks to Petr Stodulka for pointing out the discrepancy.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/plugins/upload/cmds.py
1
 
# Copyright (C) 2011, 2012 Canonical Ltd
2
 
#
3
 
# This program is free software; you can redistribute it and/or modify
4
 
# it under the terms of the GNU General Public License as published by
5
 
# the Free Software Foundation; either version 2 of the License, or
6
 
# (at your option) any later version.
7
 
#
8
 
# This program is distributed in the hope that it will be useful,
9
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
 
# GNU General Public License for more details.
12
 
#
13
 
# You should have received a copy of the GNU General Public License
14
 
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
 
 
17
 
"""bzr-upload command implementations."""
18
 
 
19
 
from __future__ import absolute_import
20
 
 
21
 
from ... import (
22
 
    branch,
23
 
    commands,
24
 
    config,
25
 
    lazy_import,
26
 
    option,
27
 
    )
28
 
lazy_import.lazy_import(globals(), """
29
 
import stat
30
 
import sys
31
 
 
32
 
from breezy import (
33
 
    controldir,
34
 
    errors,
35
 
    globbing,
36
 
    ignores,
37
 
    revision,
38
 
    transport,
39
 
    urlutils,
40
 
    )
41
 
""")
42
 
 
43
 
from ...sixish import (
44
 
    text_type,
45
 
    )
46
 
 
47
 
auto_option = config.Option(
48
 
    'upload_auto', default=False, from_unicode=config.bool_from_store,
49
 
    help="""\
50
 
Whether upload should occur when the tip of the branch changes.
51
 
""")
52
 
auto_quiet_option = config.Option(
53
 
    'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
54
 
    help="""\
55
 
Whether upload should occur quietly.
56
 
""")
57
 
location_option = config.Option(
58
 
    'upload_location', default=None,
59
 
    help="""\
60
 
The url to upload the working tree to.
61
 
""")
62
 
revid_location_option = config.Option(
63
 
    'upload_revid_location', default=u'.bzr-upload.revid',
64
 
    help="""\
65
 
The relative path to be used to store the uploaded revid.
66
 
 
67
 
The only bzr-related info uploaded with the working tree is the corresponding
68
 
revision id. The uploaded working tree is not linked to any other bzr data.
69
 
 
70
 
If the layout of your remote server is such that you can't write in the
71
 
root directory but only in the directories inside that root, you will need
72
 
to use the 'upload_revid_location' configuration variable to specify the
73
 
relative path to be used. That configuration variable can be specified in
74
 
locations.conf or branch.conf.
75
 
 
76
 
For example, given the following layout:
77
 
 
78
 
  Project/
79
 
    private/
80
 
    public/
81
 
 
82
 
you may have write access in 'private' and 'public' but in 'Project'
83
 
itself. In that case, you can add the following in your locations.conf or
84
 
branch.conf file:
85
 
 
86
 
  upload_revid_location = private/.bzr-upload.revid
87
 
""")
88
 
 
89
 
 
90
 
# FIXME: Add more tests around invalid paths or relative paths that doesn't
91
 
# exist on remote (if only to get proper error messages) for
92
 
# 'upload_revid_location'
93
 
 
94
 
 
95
 
class BzrUploader(object):
96
 
 
97
 
    def __init__(self, branch, to_transport, outf, tree, rev_id,
98
 
                 quiet=False):
99
 
        self.branch = branch
100
 
        self.to_transport = to_transport
101
 
        self.outf = outf
102
 
        self.tree = tree
103
 
        self.rev_id = rev_id
104
 
        self.quiet = quiet
105
 
        self._pending_deletions = []
106
 
        self._pending_renames = []
107
 
        self._uploaded_revid = None
108
 
        self._ignored = None
109
 
 
110
 
    def _up_stat(self, relpath):
111
 
        return self.to_transport.stat(urlutils.escape(relpath))
112
 
 
113
 
    def _up_rename(self, old_path, new_path):
114
 
        return self.to_transport.rename(urlutils.escape(old_path),
115
 
                                        urlutils.escape(new_path))
116
 
 
117
 
    def _up_delete(self, relpath):
118
 
        return self.to_transport.delete(urlutils.escape(relpath))
119
 
 
120
 
    def _up_delete_tree(self, relpath):
121
 
        return self.to_transport.delete_tree(urlutils.escape(relpath))
122
 
 
123
 
    def _up_mkdir(self, relpath, mode):
124
 
        return self.to_transport.mkdir(urlutils.escape(relpath), mode)
125
 
 
126
 
    def _up_rmdir(self, relpath):
127
 
        return self.to_transport.rmdir(urlutils.escape(relpath))
128
 
 
129
 
    def _up_put_bytes(self, relpath, bytes, mode):
130
 
        self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
131
 
 
132
 
    def _up_get_bytes(self, relpath):
133
 
        return self.to_transport.get_bytes(urlutils.escape(relpath))
134
 
 
135
 
    def set_uploaded_revid(self, rev_id):
136
 
        # XXX: Add tests for concurrent updates, etc.
137
 
        revid_path = self.branch.get_config_stack().get('upload_revid_location')
138
 
        self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
139
 
        self._uploaded_revid = rev_id
140
 
 
141
 
    def get_uploaded_revid(self):
142
 
        if self._uploaded_revid is None:
143
 
            revid_path = self.branch.get_config_stack(
144
 
                ).get('upload_revid_location')
145
 
            try:
146
 
                self._uploaded_revid = self._up_get_bytes(revid_path)
147
 
            except errors.NoSuchFile:
148
 
                # We have not uploaded to here.
149
 
                self._uploaded_revid = revision.NULL_REVISION
150
 
        return self._uploaded_revid
151
 
 
152
 
    def _get_ignored(self):
153
 
        if self._ignored is None:
154
 
            try:
155
 
                ignore_file_path = '.bzrignore-upload'
156
 
                ignore_file = self.tree.get_file(ignore_file_path)
157
 
            except errors.NoSuchFile:
158
 
                ignored_patterns = []
159
 
            else:
160
 
                ignored_patterns = ignores.parse_ignore_file(ignore_file)
161
 
            self._ignored = globbing.Globster(ignored_patterns)
162
 
        return self._ignored
163
 
 
164
 
    def is_ignored(self, relpath):
165
 
        glob = self._get_ignored()
166
 
        ignored = glob.match(relpath)
167
 
        import os
168
 
        if not ignored:
169
 
            # We still need to check that all parents are not ignored
170
 
            dir = os.path.dirname(relpath)
171
 
            while dir and not ignored:
172
 
                ignored = glob.match(dir)
173
 
                if not ignored:
174
 
                    dir = os.path.dirname(dir)
175
 
        return ignored
176
 
 
177
 
    def upload_file(self, old_relpath, new_relpath, id, mode=None):
178
 
        if mode is None:
179
 
            if self.tree.is_executable(new_relpath, id):
180
 
                mode = 0o775
181
 
            else:
182
 
                mode = 0o664
183
 
        if not self.quiet:
184
 
            self.outf.write('Uploading %s\n' % old_relpath)
185
 
        self._up_put_bytes(old_relpath, self.tree.get_file_text(new_relpath, id), mode)
186
 
 
187
 
    def upload_file_robustly(self, relpath, id, mode=None):
188
 
        """Upload a file, clearing the way on the remote side.
189
 
 
190
 
        When doing a full upload, it may happen that a directory exists where
191
 
        we want to put our file.
192
 
        """
193
 
        try:
194
 
            st = self._up_stat(relpath)
195
 
            if stat.S_ISDIR(st.st_mode):
196
 
                # A simple rmdir may not be enough
197
 
                if not self.quiet:
198
 
                    self.outf.write('Clearing %s/%s\n' % (
199
 
                            self.to_transport.external_url(), relpath))
200
 
                self._up_delete_tree(relpath)
201
 
        except errors.PathError:
202
 
            pass
203
 
        self.upload_file(relpath, relpath, id, mode)
204
 
 
205
 
    def make_remote_dir(self, relpath, mode=None):
206
 
        if mode is None:
207
 
            mode = 0o775
208
 
        self._up_mkdir(relpath, mode)
209
 
 
210
 
    def make_remote_dir_robustly(self, relpath, mode=None):
211
 
        """Create a remote directory, clearing the way on the remote side.
212
 
 
213
 
        When doing a full upload, it may happen that a file exists where we
214
 
        want to create our directory.
215
 
        """
216
 
        try:
217
 
            st = self._up_stat(relpath)
218
 
            if not stat.S_ISDIR(st.st_mode):
219
 
                if not self.quiet:
220
 
                    self.outf.write('Deleting %s/%s\n' % (
221
 
                            self.to_transport.external_url(), relpath))
222
 
                self._up_delete(relpath)
223
 
            else:
224
 
                # Ok the remote dir already exists, nothing to do
225
 
                return
226
 
        except errors.PathError:
227
 
            pass
228
 
        self.make_remote_dir(relpath, mode)
229
 
 
230
 
    def delete_remote_file(self, relpath):
231
 
        if not self.quiet:
232
 
            self.outf.write('Deleting %s\n' % relpath)
233
 
        self._up_delete(relpath)
234
 
 
235
 
    def delete_remote_dir(self, relpath):
236
 
        if not self.quiet:
237
 
            self.outf.write('Deleting %s\n' % relpath)
238
 
        self._up_rmdir(relpath)
239
 
        # XXX: Add a test where a subdir is ignored but we still want to
240
 
        # delete the dir -- vila 100106
241
 
 
242
 
    def delete_remote_dir_maybe(self, relpath):
243
 
        """Try to delete relpath, keeping failures to retry later."""
244
 
        try:
245
 
            self._up_rmdir(relpath)
246
 
        # any kind of PathError would be OK, though we normally expect
247
 
        # DirectoryNotEmpty
248
 
        except errors.PathError:
249
 
            self._pending_deletions.append(relpath)
250
 
 
251
 
    def finish_deletions(self):
252
 
        if self._pending_deletions:
253
 
            # Process the previously failed deletions in reverse order to
254
 
            # delete children before parents
255
 
            for relpath in reversed(self._pending_deletions):
256
 
                self._up_rmdir(relpath)
257
 
            # The following shouldn't be needed since we use it once per
258
 
            # upload, but better safe than sorry ;-)
259
 
            self._pending_deletions = []
260
 
 
261
 
    def rename_remote(self, old_relpath, new_relpath):
262
 
        """Rename a remote file or directory taking care of collisions.
263
 
 
264
 
        To avoid collisions during bulk renames, each renamed target is
265
 
        temporarily assigned a unique name. When all renames have been done,
266
 
        each target get its proper name.
267
 
        """
268
 
        # We generate a sufficiently random name to *assume* that
269
 
        # no collisions will occur and don't worry about it (nor
270
 
        # handle it).
271
 
        import os
272
 
        import random
273
 
        import time
274
 
 
275
 
        stamp = '.tmp.%.9f.%d.%d' % (time.time(),
276
 
                                     os.getpid(),
277
 
                                     random.randint(0, 0x7FFFFFFF))
278
 
        if not self.quiet:
279
 
            self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
280
 
        self._up_rename(old_relpath, stamp)
281
 
        self._pending_renames.append((stamp, new_relpath))
282
 
 
283
 
    def finish_renames(self):
284
 
        for (stamp, new_path) in self._pending_renames:
285
 
            self._up_rename(stamp, new_path)
286
 
        # The following shouldn't be needed since we use it once per upload,
287
 
        # but better safe than sorry ;-)
288
 
        self._pending_renames = []
289
 
 
290
 
    def upload_full_tree(self):
291
 
        self.to_transport.ensure_base() # XXX: Handle errors (add
292
 
                                        # --create-prefix option ?)
293
 
        with self.tree.lock_read():
294
 
            for relpath, ie in self.tree.iter_entries_by_dir():
295
 
                if relpath in ('', '.bzrignore', '.bzrignore-upload'):
296
 
                    # skip root ('')
297
 
                    # .bzrignore and .bzrignore-upload have no meaning outside
298
 
                    # a working tree so do not upload them
299
 
                    continue
300
 
                if self.is_ignored(relpath):
301
 
                    if not self.quiet:
302
 
                        self.outf.write('Ignoring %s\n' % relpath)
303
 
                    continue
304
 
                if ie.kind == 'file':
305
 
                    self.upload_file_robustly(relpath, ie.file_id)
306
 
                elif ie.kind == 'directory':
307
 
                    self.make_remote_dir_robustly(relpath)
308
 
                elif ie.kind == 'symlink':
309
 
                    if not self.quiet:
310
 
                        target = self.tree.path_content_summary(relpath)[3]
311
 
                        self.outf.write('Not uploading symlink %s -> %s\n'
312
 
                                        % (relpath, target))
313
 
                else:
314
 
                    raise NotImplementedError
315
 
            self.set_uploaded_revid(self.rev_id)
316
 
 
317
 
    def upload_tree(self):
318
 
        # If we can't find the revid file on the remote location, upload the
319
 
        # full tree instead
320
 
        rev_id = self.get_uploaded_revid()
321
 
 
322
 
        if rev_id == revision.NULL_REVISION:
323
 
            if not self.quiet:
324
 
                self.outf.write('No uploaded revision id found,'
325
 
                                ' switching to full upload\n')
326
 
            self.upload_full_tree()
327
 
            # We're done
328
 
            return
329
 
 
330
 
        # Check if the revision hasn't already been uploaded
331
 
        if rev_id == self.rev_id:
332
 
            if not self.quiet:
333
 
                self.outf.write('Remote location already up to date\n')
334
 
 
335
 
        from_tree = self.branch.repository.revision_tree(rev_id)
336
 
        self.to_transport.ensure_base() # XXX: Handle errors (add
337
 
                                        # --create-prefix option ?)
338
 
        changes = self.tree.changes_from(from_tree)
339
 
        with self.tree.lock_read():
340
 
            for (path, id, kind) in changes.removed:
341
 
                if self.is_ignored(path):
342
 
                    if not self.quiet:
343
 
                        self.outf.write('Ignoring %s\n' % path)
344
 
                    continue
345
 
                if kind is 'file':
346
 
                    self.delete_remote_file(path)
347
 
                elif kind is  'directory':
348
 
                    self.delete_remote_dir_maybe(path)
349
 
                elif kind == 'symlink':
350
 
                    if not self.quiet:
351
 
                        target = self.tree.path_content_summary(path)[3]
352
 
                        self.outf.write('Not deleting remote symlink %s -> %s\n'
353
 
                                        % (path, target))
354
 
                else:
355
 
                    raise NotImplementedError
356
 
 
357
 
            for (old_path, new_path, id, kind,
358
 
                 content_change, exec_change) in changes.renamed:
359
 
                if self.is_ignored(old_path) and self.is_ignored(new_path):
360
 
                    if not self.quiet:
361
 
                        self.outf.write('Ignoring %s\n' % old_path)
362
 
                        self.outf.write('Ignoring %s\n' % new_path)
363
 
                    continue
364
 
                if content_change:
365
 
                    # We update the old_path content because renames and
366
 
                    # deletions are differed.
367
 
                    self.upload_file(old_path, new_path, id)
368
 
                if kind == 'symlink':
369
 
                    if not self.quiet:
370
 
                        self.outf.write('Not renaming remote symlink %s to %s\n'
371
 
                                        % (old_path, new_path))
372
 
                else:
373
 
                    self.rename_remote(old_path, new_path)
374
 
            self.finish_renames()
375
 
            self.finish_deletions()
376
 
 
377
 
            for (path, id, old_kind, new_kind) in changes.kind_changed:
378
 
                if self.is_ignored(path):
379
 
                    if not self.quiet:
380
 
                        self.outf.write('Ignoring %s\n' % path)
381
 
                    continue
382
 
                if old_kind == 'file':
383
 
                    self.delete_remote_file(path)
384
 
                elif old_kind ==  'directory':
385
 
                    self.delete_remote_dir(path)
386
 
                else:
387
 
                    raise NotImplementedError
388
 
 
389
 
                if new_kind == 'file':
390
 
                    self.upload_file(path, path, id)
391
 
                elif new_kind is 'directory':
392
 
                    self.make_remote_dir(path)
393
 
                else:
394
 
                    raise NotImplementedError
395
 
 
396
 
            for (path, id, kind) in changes.added:
397
 
                if self.is_ignored(path):
398
 
                    if not self.quiet:
399
 
                        self.outf.write('Ignoring %s\n' % path)
400
 
                    continue
401
 
                if kind == 'file':
402
 
                    self.upload_file(path, path, id)
403
 
                elif kind == 'directory':
404
 
                    self.make_remote_dir(path)
405
 
                elif kind == 'symlink':
406
 
                    if not self.quiet:
407
 
                        target = self.tree.path_content_summary(path)[3]
408
 
                        self.outf.write('Not uploading symlink %s -> %s\n'
409
 
                                        % (path, target))
410
 
                else:
411
 
                    raise NotImplementedError
412
 
 
413
 
            # XXX: Add a test for exec_change
414
 
            for (path, id, kind,
415
 
                 content_change, exec_change) in changes.modified:
416
 
                if self.is_ignored(path):
417
 
                    if not self.quiet:
418
 
                        self.outf.write('Ignoring %s\n' % path)
419
 
                    continue
420
 
                if kind == 'file':
421
 
                    self.upload_file(path, path, id)
422
 
                else:
423
 
                    raise NotImplementedError
424
 
 
425
 
            self.set_uploaded_revid(self.rev_id)
426
 
 
427
 
 
428
 
class CannotUploadToWorkingTree(errors.BzrCommandError):
429
 
 
430
 
    _fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
431
 
 
432
 
 
433
 
class DivergedUploadedTree(errors.BzrCommandError):
434
 
 
435
 
    _fmt = ("Your branch (%(revid)s)"
436
 
            " and the uploaded tree (%(uploaded_revid)s) have diverged: ")
437
 
 
438
 
 
439
 
class cmd_upload(commands.Command):
440
 
    """Upload a working tree, as a whole or incrementally.
441
 
 
442
 
    If no destination is specified use the last one used.
443
 
    If no revision is specified upload the changes since the last upload.
444
 
 
445
 
    Changes include files added, renamed, modified or removed.
446
 
    """
447
 
    _see_also = ['plugins/upload']
448
 
    takes_args = ['location?']
449
 
    takes_options = [
450
 
        'revision',
451
 
        'remember',
452
 
        'overwrite',
453
 
        option.Option('full', 'Upload the full working tree.'),
454
 
        option.Option('quiet', 'Do not output what is being done.',
455
 
                       short_name='q'),
456
 
        option.Option('directory',
457
 
                      help='Branch to upload from, '
458
 
                      'rather than the one containing the working directory.',
459
 
                      short_name='d',
460
 
                      type=text_type,
461
 
                      ),
462
 
        option.Option('auto',
463
 
                      'Trigger an upload from this branch whenever the tip '
464
 
                      'revision changes.')
465
 
       ]
466
 
 
467
 
    def run(self, location=None, full=False, revision=None, remember=None,
468
 
            directory=None, quiet=False, auto=None, overwrite=False
469
 
            ):
470
 
        if directory is None:
471
 
            directory = u'.'
472
 
 
473
 
        (wt, branch,
474
 
         relpath) = controldir.ControlDir.open_containing_tree_or_branch(
475
 
             directory)
476
 
 
477
 
        if wt:
478
 
            wt.lock_read()
479
 
            locked = wt
480
 
        else:
481
 
            branch.lock_read()
482
 
            locked = branch
483
 
        try:
484
 
            if wt:
485
 
                changes = wt.changes_from(wt.basis_tree())
486
 
 
487
 
                if revision is None and  changes.has_changed():
488
 
                    raise errors.UncommittedChanges(wt)
489
 
 
490
 
            conf = branch.get_config_stack()
491
 
            if location is None:
492
 
                stored_loc = conf.get('upload_location')
493
 
                if stored_loc is None:
494
 
                    raise errors.BzrCommandError(
495
 
                        'No upload location known or specified.')
496
 
                else:
497
 
                    # FIXME: Not currently tested
498
 
                    display_url = urlutils.unescape_for_display(stored_loc,
499
 
                            self.outf.encoding)
500
 
                    self.outf.write("Using saved location: %s\n" % display_url)
501
 
                    location = stored_loc
502
 
 
503
 
            to_transport = transport.get_transport(location)
504
 
 
505
 
            # Check that we are not uploading to a existing working tree.
506
 
            try:
507
 
                to_bzr_dir = controldir.ControlDir.open_from_transport(
508
 
                        to_transport)
509
 
                has_wt = to_bzr_dir.has_workingtree()
510
 
            except errors.NotBranchError:
511
 
                has_wt = False
512
 
            except errors.NotLocalUrl:
513
 
                # The exception raised is a bit weird... but that's life.
514
 
                has_wt = True
515
 
 
516
 
            if has_wt:
517
 
                raise CannotUploadToWorkingTree(url=location)
518
 
            if revision is None:
519
 
                rev_id = branch.last_revision()
520
 
            else:
521
 
                if len(revision) != 1:
522
 
                    raise errors.BzrCommandError(
523
 
                        'bzr upload --revision takes exactly 1 argument')
524
 
                rev_id = revision[0].in_history(branch).rev_id
525
 
 
526
 
            tree = branch.repository.revision_tree(rev_id)
527
 
 
528
 
            uploader = BzrUploader(branch, to_transport, self.outf, tree,
529
 
                                   rev_id, quiet=quiet)
530
 
 
531
 
            if not overwrite:
532
 
                prev_uploaded_rev_id = uploader.get_uploaded_revid()
533
 
                graph = branch.repository.get_graph()
534
 
                if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
535
 
                    raise DivergedUploadedTree(
536
 
                        revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
537
 
            if full:
538
 
                uploader.upload_full_tree()
539
 
            else:
540
 
                uploader.upload_tree()
541
 
        finally:
542
 
            locked.unlock()
543
 
 
544
 
        # We uploaded successfully, remember it
545
 
        branch.lock_write()
546
 
        try:
547
 
            upload_location = conf.get('upload_location')
548
 
            if upload_location is None or remember:
549
 
                conf.set('upload_location',
550
 
                         urlutils.unescape(to_transport.base))
551
 
            if auto is not None:
552
 
                conf.set('upload_auto', auto)
553
 
        finally:
554
 
            branch.unlock()
555